1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file performs vector type splitting and scalarization for LegalizeTypes. 11 // Scalarization is the act of changing a computation in an illegal one-element 12 // vector type to be a computation in its scalar element type. For example, 13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which 15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16 // types. 17 // Splitting is the act of changing a computation in an invalid vector type to 18 // be a computation in two vectors of half the size. For example, implementing 19 // <128 x f32> operations in terms of two <64 x f32> operations. 20 // 21 //===----------------------------------------------------------------------===// 22 23 #include "LegalizeTypes.h" 24 #include "llvm/IR/DataLayout.h" 25 #include "llvm/Support/ErrorHandling.h" 26 #include "llvm/Support/raw_ostream.h" 27 using namespace llvm; 28 29 #define DEBUG_TYPE "legalize-types" 30 31 //===----------------------------------------------------------------------===// 32 // Result Vector Scalarization: <1 x ty> -> ty. 33 //===----------------------------------------------------------------------===// 34 35 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 36 DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; 37 N->dump(&DAG); 38 dbgs() << "\n"); 39 SDValue R = SDValue(); 40 41 switch (N->getOpcode()) { 42 default: 43 #ifndef NDEBUG 44 dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 45 N->dump(&DAG); 46 dbgs() << "\n"; 47 #endif 48 report_fatal_error("Do not know how to scalarize the result of this " 49 "operator!\n"); 50 51 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; 52 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; 53 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; 54 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 55 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 56 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; 57 case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break; 58 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 59 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 60 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 61 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 62 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 63 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break; 64 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 65 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 66 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 67 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 68 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 69 case ISD::ANY_EXTEND: 70 case ISD::BSWAP: 71 case ISD::CTLZ: 72 case ISD::CTLZ_ZERO_UNDEF: 73 case ISD::CTPOP: 74 case ISD::CTTZ: 75 case ISD::CTTZ_ZERO_UNDEF: 76 case ISD::FABS: 77 case ISD::FCEIL: 78 case ISD::FCOS: 79 case ISD::FEXP: 80 case ISD::FEXP2: 81 case ISD::FFLOOR: 82 case ISD::FLOG: 83 case ISD::FLOG10: 84 case ISD::FLOG2: 85 case ISD::FNEARBYINT: 86 case ISD::FNEG: 87 case ISD::FP_EXTEND: 88 case ISD::FP_TO_SINT: 89 case ISD::FP_TO_UINT: 90 case ISD::FRINT: 91 case ISD::FROUND: 92 case ISD::FSIN: 93 case ISD::FSQRT: 94 case ISD::FTRUNC: 95 case ISD::SIGN_EXTEND: 96 case ISD::SINT_TO_FP: 97 case ISD::TRUNCATE: 98 case ISD::UINT_TO_FP: 99 case ISD::ZERO_EXTEND: 100 R = ScalarizeVecRes_UnaryOp(N); 101 break; 102 103 case ISD::ADD: 104 case ISD::AND: 105 case ISD::FADD: 106 case ISD::FCOPYSIGN: 107 case ISD::FDIV: 108 case ISD::FMUL: 109 case ISD::FPOW: 110 case ISD::FREM: 111 case ISD::FSUB: 112 case ISD::MUL: 113 case ISD::OR: 114 case ISD::SDIV: 115 case ISD::SREM: 116 case ISD::SUB: 117 case ISD::UDIV: 118 case ISD::UREM: 119 case ISD::XOR: 120 case ISD::SHL: 121 case ISD::SRA: 122 case ISD::SRL: 123 R = ScalarizeVecRes_BinOp(N); 124 break; 125 case ISD::FMA: 126 R = ScalarizeVecRes_TernaryOp(N); 127 break; 128 } 129 130 // If R is null, the sub-method took care of registering the result. 131 if (R.getNode()) 132 SetScalarizedVector(SDValue(N, ResNo), R); 133 } 134 135 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 136 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 137 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 138 return DAG.getNode(N->getOpcode(), SDLoc(N), 139 LHS.getValueType(), LHS, RHS); 140 } 141 142 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { 143 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 144 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 145 SDValue Op2 = GetScalarizedVector(N->getOperand(2)); 146 return DAG.getNode(N->getOpcode(), SDLoc(N), 147 Op0.getValueType(), Op0, Op1, Op2); 148 } 149 150 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, 151 unsigned ResNo) { 152 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 153 return GetScalarizedVector(Op); 154 } 155 156 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { 157 EVT NewVT = N->getValueType(0).getVectorElementType(); 158 return DAG.getNode(ISD::BITCAST, SDLoc(N), 159 NewVT, N->getOperand(0)); 160 } 161 162 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { 163 EVT EltVT = N->getValueType(0).getVectorElementType(); 164 SDValue InOp = N->getOperand(0); 165 // The BUILD_VECTOR operands may be of wider element types and 166 // we may need to truncate them back to the requested return type. 167 if (EltVT.isInteger()) 168 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 169 return InOp; 170 } 171 172 SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 173 EVT NewVT = N->getValueType(0).getVectorElementType(); 174 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 175 return DAG.getConvertRndSat(NewVT, SDLoc(N), 176 Op0, DAG.getValueType(NewVT), 177 DAG.getValueType(Op0.getValueType()), 178 N->getOperand(3), 179 N->getOperand(4), 180 cast<CvtRndSatSDNode>(N)->getCvtCode()); 181 } 182 183 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 184 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 185 N->getValueType(0).getVectorElementType(), 186 N->getOperand(0), N->getOperand(1)); 187 } 188 189 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { 190 EVT NewVT = N->getValueType(0).getVectorElementType(); 191 SDValue Op = GetScalarizedVector(N->getOperand(0)); 192 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 193 NewVT, Op, N->getOperand(1)); 194 } 195 196 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 197 SDValue Op = GetScalarizedVector(N->getOperand(0)); 198 return DAG.getNode(ISD::FPOWI, SDLoc(N), 199 Op.getValueType(), Op, N->getOperand(1)); 200 } 201 202 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 203 // The value to insert may have a wider type than the vector element type, 204 // so be sure to truncate it to the element type if necessary. 205 SDValue Op = N->getOperand(1); 206 EVT EltVT = N->getValueType(0).getVectorElementType(); 207 if (Op.getValueType() != EltVT) 208 // FIXME: Can this happen for floating point types? 209 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op); 210 return Op; 211 } 212 213 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 214 assert(N->isUnindexed() && "Indexed vector load?"); 215 216 SDValue Result = DAG.getLoad(ISD::UNINDEXED, 217 N->getExtensionType(), 218 N->getValueType(0).getVectorElementType(), 219 SDLoc(N), 220 N->getChain(), N->getBasePtr(), 221 DAG.getUNDEF(N->getBasePtr().getValueType()), 222 N->getPointerInfo(), 223 N->getMemoryVT().getVectorElementType(), 224 N->isVolatile(), N->isNonTemporal(), 225 N->isInvariant(), N->getOriginalAlignment(), 226 N->getTBAAInfo()); 227 228 // Legalized the chain result - switch anything that used the old chain to 229 // use the new one. 230 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 231 return Result; 232 } 233 234 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 235 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 236 EVT DestVT = N->getValueType(0).getVectorElementType(); 237 SDValue Op = GetScalarizedVector(N->getOperand(0)); 238 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op); 239 } 240 241 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 242 EVT EltVT = N->getValueType(0).getVectorElementType(); 243 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 244 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 245 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT, 246 LHS, DAG.getValueType(ExtVT)); 247 } 248 249 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 250 // If the operand is wider than the vector element type then it is implicitly 251 // truncated. Make that explicit here. 252 EVT EltVT = N->getValueType(0).getVectorElementType(); 253 SDValue InOp = N->getOperand(0); 254 if (InOp.getValueType() != EltVT) 255 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 256 return InOp; 257 } 258 259 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { 260 SDValue Cond = GetScalarizedVector(N->getOperand(0)); 261 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 262 TargetLowering::BooleanContent ScalarBool = 263 TLI.getBooleanContents(false, false); 264 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false); 265 266 // If integer and float booleans have different contents then we can't 267 // reliably optimize in all cases. There is a full explanation for this in 268 // DAGCombiner::visitSELECT() where the same issue affects folding 269 // (select C, 0, 1) to (xor C, 1). 270 if (TLI.getBooleanContents(false, false) != 271 TLI.getBooleanContents(false, true)) { 272 // At least try the common case where the boolean is generated by a 273 // comparison. 274 if (Cond->getOpcode() == ISD::SETCC) { 275 EVT OpVT = Cond->getOperand(0)->getValueType(0); 276 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType()); 277 VecBool = TLI.getBooleanContents(OpVT); 278 } else 279 ScalarBool = TargetLowering::UndefinedBooleanContent; 280 } 281 282 if (ScalarBool != VecBool) { 283 EVT CondVT = Cond.getValueType(); 284 switch (ScalarBool) { 285 case TargetLowering::UndefinedBooleanContent: 286 break; 287 case TargetLowering::ZeroOrOneBooleanContent: 288 assert(VecBool == TargetLowering::UndefinedBooleanContent || 289 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent); 290 // Vector read from all ones, scalar expects a single 1 so mask. 291 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT, 292 Cond, DAG.getConstant(1, CondVT)); 293 break; 294 case TargetLowering::ZeroOrNegativeOneBooleanContent: 295 assert(VecBool == TargetLowering::UndefinedBooleanContent || 296 VecBool == TargetLowering::ZeroOrOneBooleanContent); 297 // Vector reads from a one, scalar from all ones so sign extend. 298 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT, 299 Cond, DAG.getValueType(MVT::i1)); 300 break; 301 } 302 } 303 304 return DAG.getSelect(SDLoc(N), 305 LHS.getValueType(), Cond, LHS, 306 GetScalarizedVector(N->getOperand(2))); 307 } 308 309 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 310 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 311 return DAG.getSelect(SDLoc(N), 312 LHS.getValueType(), N->getOperand(0), LHS, 313 GetScalarizedVector(N->getOperand(2))); 314 } 315 316 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 317 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 318 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(), 319 N->getOperand(0), N->getOperand(1), 320 LHS, GetScalarizedVector(N->getOperand(3)), 321 N->getOperand(4)); 322 } 323 324 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 325 assert(N->getValueType(0).isVector() == 326 N->getOperand(0).getValueType().isVector() && 327 "Scalar/Vector type mismatch"); 328 329 if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N); 330 331 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 332 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 333 SDLoc DL(N); 334 335 // Turn it into a scalar SETCC. 336 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 337 } 338 339 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 340 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 341 } 342 343 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 344 // Figure out if the scalar is the LHS or RHS and return it. 345 SDValue Arg = N->getOperand(2).getOperand(0); 346 if (Arg.getOpcode() == ISD::UNDEF) 347 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 348 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 349 return GetScalarizedVector(N->getOperand(Op)); 350 } 351 352 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 353 assert(N->getValueType(0).isVector() && 354 N->getOperand(0).getValueType().isVector() && 355 "Operand types must be vectors"); 356 SDValue LHS = N->getOperand(0); 357 SDValue RHS = N->getOperand(1); 358 EVT OpVT = LHS.getValueType(); 359 EVT NVT = N->getValueType(0).getVectorElementType(); 360 SDLoc DL(N); 361 362 // The result needs scalarizing, but it's not a given that the source does. 363 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 364 LHS = GetScalarizedVector(LHS); 365 RHS = GetScalarizedVector(RHS); 366 } else { 367 EVT VT = OpVT.getVectorElementType(); 368 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS, 369 DAG.getConstant(0, TLI.getVectorIdxTy())); 370 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS, 371 DAG.getConstant(0, TLI.getVectorIdxTy())); 372 } 373 374 // Turn it into a scalar SETCC. 375 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 376 N->getOperand(2)); 377 // Vectors may have a different boolean contents to scalars. Promote the 378 // value appropriately. 379 ISD::NodeType ExtendCode = 380 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 381 return DAG.getNode(ExtendCode, DL, NVT, Res); 382 } 383 384 385 //===----------------------------------------------------------------------===// 386 // Operand Vector Scalarization <1 x ty> -> ty. 387 //===----------------------------------------------------------------------===// 388 389 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 390 DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; 391 N->dump(&DAG); 392 dbgs() << "\n"); 393 SDValue Res = SDValue(); 394 395 if (!Res.getNode()) { 396 switch (N->getOpcode()) { 397 default: 398 #ifndef NDEBUG 399 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 400 N->dump(&DAG); 401 dbgs() << "\n"; 402 #endif 403 llvm_unreachable("Do not know how to scalarize this operator's operand!"); 404 case ISD::BITCAST: 405 Res = ScalarizeVecOp_BITCAST(N); 406 break; 407 case ISD::ANY_EXTEND: 408 case ISD::ZERO_EXTEND: 409 case ISD::SIGN_EXTEND: 410 case ISD::TRUNCATE: 411 Res = ScalarizeVecOp_UnaryOp(N); 412 break; 413 case ISD::CONCAT_VECTORS: 414 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 415 break; 416 case ISD::EXTRACT_VECTOR_ELT: 417 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 418 break; 419 case ISD::VSELECT: 420 Res = ScalarizeVecOp_VSELECT(N); 421 break; 422 case ISD::STORE: 423 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 424 break; 425 case ISD::FP_ROUND: 426 Res = ScalarizeVecOp_FP_ROUND(N, OpNo); 427 break; 428 } 429 } 430 431 // If the result is null, the sub-method took care of registering results etc. 432 if (!Res.getNode()) return false; 433 434 // If the result is N, the sub-method updated N in place. Tell the legalizer 435 // core about this. 436 if (Res.getNode() == N) 437 return true; 438 439 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 440 "Invalid operand expansion"); 441 442 ReplaceValueWith(SDValue(N, 0), Res); 443 return false; 444 } 445 446 /// ScalarizeVecOp_BITCAST - If the value to convert is a vector that needs 447 /// to be scalarized, it must be <1 x ty>. Convert the element instead. 448 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { 449 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 450 return DAG.getNode(ISD::BITCAST, SDLoc(N), 451 N->getValueType(0), Elt); 452 } 453 454 /// ScalarizeVecOp_EXTEND - If the value to extend is a vector that needs 455 /// to be scalarized, it must be <1 x ty>. Extend the element instead. 456 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) { 457 assert(N->getValueType(0).getVectorNumElements() == 1 && 458 "Unexected vector type!"); 459 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 460 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N), 461 N->getValueType(0).getScalarType(), Elt); 462 // Revectorize the result so the types line up with what the uses of this 463 // expression expect. 464 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), Op); 465 } 466 467 /// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 468 /// use a BUILD_VECTOR instead. 469 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 470 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 471 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 472 Ops[i] = GetScalarizedVector(N->getOperand(i)); 473 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), Ops); 474 } 475 476 /// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 477 /// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 478 /// index. 479 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 480 SDValue Res = GetScalarizedVector(N->getOperand(0)); 481 if (Res.getValueType() != N->getValueType(0)) 482 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), 483 Res); 484 return Res; 485 } 486 487 488 /// ScalarizeVecOp_VSELECT - If the input condition is a vector that needs to be 489 /// scalarized, it must be <1 x i1>, so just convert to a normal ISD::SELECT 490 /// (still with vector output type since that was acceptable if we got here). 491 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) { 492 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0)); 493 EVT VT = N->getValueType(0); 494 495 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1), 496 N->getOperand(2)); 497 } 498 499 /// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 500 /// scalarized, it must be <1 x ty>. Just store the element. 501 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 502 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 503 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 504 SDLoc dl(N); 505 506 if (N->isTruncatingStore()) 507 return DAG.getTruncStore(N->getChain(), dl, 508 GetScalarizedVector(N->getOperand(1)), 509 N->getBasePtr(), N->getPointerInfo(), 510 N->getMemoryVT().getVectorElementType(), 511 N->isVolatile(), N->isNonTemporal(), 512 N->getAlignment(), N->getTBAAInfo()); 513 514 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 515 N->getBasePtr(), N->getPointerInfo(), 516 N->isVolatile(), N->isNonTemporal(), 517 N->getOriginalAlignment(), N->getTBAAInfo()); 518 } 519 520 /// ScalarizeVecOp_FP_ROUND - If the value to round is a vector that needs 521 /// to be scalarized, it must be <1 x ty>. Convert the element instead. 522 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) { 523 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 524 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N), 525 N->getValueType(0).getVectorElementType(), Elt, 526 N->getOperand(1)); 527 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 528 } 529 530 //===----------------------------------------------------------------------===// 531 // Result Vector Splitting 532 //===----------------------------------------------------------------------===// 533 534 /// SplitVectorResult - This method is called when the specified result of the 535 /// specified node is found to need vector splitting. At this point, the node 536 /// may also have invalid operands or may have other results that need 537 /// legalization, we just know that (at least) one result needs vector 538 /// splitting. 539 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 540 DEBUG(dbgs() << "Split node result: "; 541 N->dump(&DAG); 542 dbgs() << "\n"); 543 SDValue Lo, Hi; 544 545 // See if the target wants to custom expand this node. 546 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 547 return; 548 549 switch (N->getOpcode()) { 550 default: 551 #ifndef NDEBUG 552 dbgs() << "SplitVectorResult #" << ResNo << ": "; 553 N->dump(&DAG); 554 dbgs() << "\n"; 555 #endif 556 report_fatal_error("Do not know how to split the result of this " 557 "operator!\n"); 558 559 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 560 case ISD::VSELECT: 561 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 562 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 563 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 564 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break; 565 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 566 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 567 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 568 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break; 569 case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 570 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 571 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 572 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 573 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 574 case ISD::LOAD: 575 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 576 break; 577 case ISD::SETCC: 578 SplitVecRes_SETCC(N, Lo, Hi); 579 break; 580 case ISD::VECTOR_SHUFFLE: 581 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 582 break; 583 584 case ISD::BSWAP: 585 case ISD::CONVERT_RNDSAT: 586 case ISD::CTLZ: 587 case ISD::CTTZ: 588 case ISD::CTLZ_ZERO_UNDEF: 589 case ISD::CTTZ_ZERO_UNDEF: 590 case ISD::CTPOP: 591 case ISD::FABS: 592 case ISD::FCEIL: 593 case ISD::FCOS: 594 case ISD::FEXP: 595 case ISD::FEXP2: 596 case ISD::FFLOOR: 597 case ISD::FLOG: 598 case ISD::FLOG10: 599 case ISD::FLOG2: 600 case ISD::FNEARBYINT: 601 case ISD::FNEG: 602 case ISD::FP_EXTEND: 603 case ISD::FP_ROUND: 604 case ISD::FP_TO_SINT: 605 case ISD::FP_TO_UINT: 606 case ISD::FRINT: 607 case ISD::FROUND: 608 case ISD::FSIN: 609 case ISD::FSQRT: 610 case ISD::FTRUNC: 611 case ISD::SINT_TO_FP: 612 case ISD::TRUNCATE: 613 case ISD::UINT_TO_FP: 614 SplitVecRes_UnaryOp(N, Lo, Hi); 615 break; 616 617 case ISD::ANY_EXTEND: 618 case ISD::SIGN_EXTEND: 619 case ISD::ZERO_EXTEND: 620 SplitVecRes_ExtendOp(N, Lo, Hi); 621 break; 622 623 case ISD::ADD: 624 case ISD::SUB: 625 case ISD::MUL: 626 case ISD::FADD: 627 case ISD::FCOPYSIGN: 628 case ISD::FSUB: 629 case ISD::FMUL: 630 case ISD::SDIV: 631 case ISD::UDIV: 632 case ISD::FDIV: 633 case ISD::FPOW: 634 case ISD::AND: 635 case ISD::OR: 636 case ISD::XOR: 637 case ISD::SHL: 638 case ISD::SRA: 639 case ISD::SRL: 640 case ISD::UREM: 641 case ISD::SREM: 642 case ISD::FREM: 643 SplitVecRes_BinOp(N, Lo, Hi); 644 break; 645 case ISD::FMA: 646 SplitVecRes_TernaryOp(N, Lo, Hi); 647 break; 648 } 649 650 // If Lo/Hi is null, the sub-method took care of registering results etc. 651 if (Lo.getNode()) 652 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 653 } 654 655 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 656 SDValue &Hi) { 657 SDValue LHSLo, LHSHi; 658 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 659 SDValue RHSLo, RHSHi; 660 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 661 SDLoc dl(N); 662 663 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 664 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 665 } 666 667 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, 668 SDValue &Hi) { 669 SDValue Op0Lo, Op0Hi; 670 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 671 SDValue Op1Lo, Op1Hi; 672 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 673 SDValue Op2Lo, Op2Hi; 674 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi); 675 SDLoc dl(N); 676 677 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), 678 Op0Lo, Op1Lo, Op2Lo); 679 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), 680 Op0Hi, Op1Hi, Op2Hi); 681 } 682 683 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, 684 SDValue &Hi) { 685 // We know the result is a vector. The input may be either a vector or a 686 // scalar value. 687 EVT LoVT, HiVT; 688 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 689 SDLoc dl(N); 690 691 SDValue InOp = N->getOperand(0); 692 EVT InVT = InOp.getValueType(); 693 694 // Handle some special cases efficiently. 695 switch (getTypeAction(InVT)) { 696 case TargetLowering::TypeLegal: 697 case TargetLowering::TypePromoteInteger: 698 case TargetLowering::TypeSoftenFloat: 699 case TargetLowering::TypeScalarizeVector: 700 case TargetLowering::TypeWidenVector: 701 break; 702 case TargetLowering::TypeExpandInteger: 703 case TargetLowering::TypeExpandFloat: 704 // A scalar to vector conversion, where the scalar needs expansion. 705 // If the vector is being split in two then we can just convert the 706 // expanded pieces. 707 if (LoVT == HiVT) { 708 GetExpandedOp(InOp, Lo, Hi); 709 if (TLI.isBigEndian()) 710 std::swap(Lo, Hi); 711 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 712 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 713 return; 714 } 715 break; 716 case TargetLowering::TypeSplitVector: 717 // If the input is a vector that needs to be split, convert each split 718 // piece of the input now. 719 GetSplitVector(InOp, Lo, Hi); 720 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 721 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 722 return; 723 } 724 725 // In the general case, convert the input to an integer and split it by hand. 726 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 727 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 728 if (TLI.isBigEndian()) 729 std::swap(LoIntVT, HiIntVT); 730 731 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 732 733 if (TLI.isBigEndian()) 734 std::swap(Lo, Hi); 735 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 736 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 737 } 738 739 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 740 SDValue &Hi) { 741 EVT LoVT, HiVT; 742 SDLoc dl(N); 743 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 744 unsigned LoNumElts = LoVT.getVectorNumElements(); 745 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 746 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, LoOps); 747 748 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 749 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, HiOps); 750 } 751 752 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 753 SDValue &Hi) { 754 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 755 SDLoc dl(N); 756 unsigned NumSubvectors = N->getNumOperands() / 2; 757 if (NumSubvectors == 1) { 758 Lo = N->getOperand(0); 759 Hi = N->getOperand(1); 760 return; 761 } 762 763 EVT LoVT, HiVT; 764 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 765 766 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 767 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps); 768 769 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 770 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps); 771 } 772 773 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 774 SDValue &Hi) { 775 SDValue Vec = N->getOperand(0); 776 SDValue Idx = N->getOperand(1); 777 SDLoc dl(N); 778 779 EVT LoVT, HiVT; 780 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 781 782 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 783 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 784 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, 785 DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), 786 TLI.getVectorIdxTy())); 787 } 788 789 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, 790 SDValue &Hi) { 791 SDValue Vec = N->getOperand(0); 792 SDValue SubVec = N->getOperand(1); 793 SDValue Idx = N->getOperand(2); 794 SDLoc dl(N); 795 GetSplitVector(Vec, Lo, Hi); 796 797 // Spill the vector to the stack. 798 EVT VecVT = Vec.getValueType(); 799 EVT SubVecVT = VecVT.getVectorElementType(); 800 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 801 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 802 MachinePointerInfo(), false, false, 0); 803 804 // Store the new subvector into the specified index. 805 SDValue SubVecPtr = GetVectorElementPointer(StackPtr, SubVecVT, Idx); 806 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 807 unsigned Alignment = TLI.getDataLayout()->getPrefTypeAlignment(VecType); 808 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo(), 809 false, false, 0); 810 811 // Load the Lo part from the stack slot. 812 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 813 false, false, false, 0); 814 815 // Increment the pointer to the other part. 816 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 817 StackPtr = 818 DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 819 DAG.getConstant(IncrementSize, StackPtr.getValueType())); 820 821 // Load the Hi part from the stack slot. 822 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 823 false, false, false, MinAlign(Alignment, IncrementSize)); 824 } 825 826 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 827 SDValue &Hi) { 828 SDLoc dl(N); 829 GetSplitVector(N->getOperand(0), Lo, Hi); 830 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 831 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 832 } 833 834 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 835 SDValue &Hi) { 836 SDValue LHSLo, LHSHi; 837 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 838 SDLoc dl(N); 839 840 EVT LoVT, HiVT; 841 std::tie(LoVT, HiVT) = 842 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT()); 843 844 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 845 DAG.getValueType(LoVT)); 846 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 847 DAG.getValueType(HiVT)); 848 } 849 850 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 851 SDValue &Hi) { 852 SDValue Vec = N->getOperand(0); 853 SDValue Elt = N->getOperand(1); 854 SDValue Idx = N->getOperand(2); 855 SDLoc dl(N); 856 GetSplitVector(Vec, Lo, Hi); 857 858 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 859 unsigned IdxVal = CIdx->getZExtValue(); 860 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 861 if (IdxVal < LoNumElts) 862 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 863 Lo.getValueType(), Lo, Elt, Idx); 864 else 865 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 866 DAG.getConstant(IdxVal - LoNumElts, 867 TLI.getVectorIdxTy())); 868 return; 869 } 870 871 // Spill the vector to the stack. 872 EVT VecVT = Vec.getValueType(); 873 EVT EltVT = VecVT.getVectorElementType(); 874 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 875 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 876 MachinePointerInfo(), false, false, 0); 877 878 // Store the new element. This may be larger than the vector element type, 879 // so use a truncating store. 880 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 881 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 882 unsigned Alignment = 883 TLI.getDataLayout()->getPrefTypeAlignment(VecType); 884 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT, 885 false, false, 0); 886 887 // Load the Lo part from the stack slot. 888 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 889 false, false, false, 0); 890 891 // Increment the pointer to the other part. 892 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 893 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 894 DAG.getConstant(IncrementSize, StackPtr.getValueType())); 895 896 // Load the Hi part from the stack slot. 897 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 898 false, false, false, MinAlign(Alignment, IncrementSize)); 899 } 900 901 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 902 SDValue &Hi) { 903 EVT LoVT, HiVT; 904 SDLoc dl(N); 905 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 906 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 907 Hi = DAG.getUNDEF(HiVT); 908 } 909 910 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 911 SDValue &Hi) { 912 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 913 EVT LoVT, HiVT; 914 SDLoc dl(LD); 915 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0)); 916 917 ISD::LoadExtType ExtType = LD->getExtensionType(); 918 SDValue Ch = LD->getChain(); 919 SDValue Ptr = LD->getBasePtr(); 920 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 921 EVT MemoryVT = LD->getMemoryVT(); 922 unsigned Alignment = LD->getOriginalAlignment(); 923 bool isVolatile = LD->isVolatile(); 924 bool isNonTemporal = LD->isNonTemporal(); 925 bool isInvariant = LD->isInvariant(); 926 const MDNode *TBAAInfo = LD->getTBAAInfo(); 927 928 EVT LoMemVT, HiMemVT; 929 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 930 931 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset, 932 LD->getPointerInfo(), LoMemVT, isVolatile, isNonTemporal, 933 isInvariant, Alignment, TBAAInfo); 934 935 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 936 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 937 DAG.getConstant(IncrementSize, Ptr.getValueType())); 938 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, 939 LD->getPointerInfo().getWithOffset(IncrementSize), 940 HiMemVT, isVolatile, isNonTemporal, isInvariant, Alignment, 941 TBAAInfo); 942 943 // Build a factor node to remember that this load is independent of the 944 // other one. 945 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 946 Hi.getValue(1)); 947 948 // Legalized the chain result - switch anything that used the old chain to 949 // use the new one. 950 ReplaceValueWith(SDValue(LD, 1), Ch); 951 } 952 953 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 954 assert(N->getValueType(0).isVector() && 955 N->getOperand(0).getValueType().isVector() && 956 "Operand types must be vectors"); 957 958 EVT LoVT, HiVT; 959 SDLoc DL(N); 960 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 961 962 // Split the input. 963 SDValue LL, LH, RL, RH; 964 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0); 965 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1); 966 967 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 968 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 969 } 970 971 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 972 SDValue &Hi) { 973 // Get the dest types - they may not match the input types, e.g. int_to_fp. 974 EVT LoVT, HiVT; 975 SDLoc dl(N); 976 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 977 978 // If the input also splits, handle it directly for a compile time speedup. 979 // Otherwise split it by hand. 980 EVT InVT = N->getOperand(0).getValueType(); 981 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) 982 GetSplitVector(N->getOperand(0), Lo, Hi); 983 else 984 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); 985 986 if (N->getOpcode() == ISD::FP_ROUND) { 987 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1)); 988 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1)); 989 } else if (N->getOpcode() == ISD::CONVERT_RNDSAT) { 990 SDValue DTyOpLo = DAG.getValueType(LoVT); 991 SDValue DTyOpHi = DAG.getValueType(HiVT); 992 SDValue STyOpLo = DAG.getValueType(Lo.getValueType()); 993 SDValue STyOpHi = DAG.getValueType(Hi.getValueType()); 994 SDValue RndOp = N->getOperand(3); 995 SDValue SatOp = N->getOperand(4); 996 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 997 Lo = DAG.getConvertRndSat(LoVT, dl, Lo, DTyOpLo, STyOpLo, RndOp, SatOp, 998 CvtCode); 999 Hi = DAG.getConvertRndSat(HiVT, dl, Hi, DTyOpHi, STyOpHi, RndOp, SatOp, 1000 CvtCode); 1001 } else { 1002 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 1003 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 1004 } 1005 } 1006 1007 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, 1008 SDValue &Hi) { 1009 SDLoc dl(N); 1010 EVT SrcVT = N->getOperand(0).getValueType(); 1011 EVT DestVT = N->getValueType(0); 1012 EVT LoVT, HiVT; 1013 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT); 1014 1015 // We can do better than a generic split operation if the extend is doing 1016 // more than just doubling the width of the elements and the following are 1017 // true: 1018 // - The number of vector elements is even, 1019 // - the source type is legal, 1020 // - the type of a split source is illegal, 1021 // - the type of an extended (by doubling element size) source is legal, and 1022 // - the type of that extended source when split is legal. 1023 // 1024 // This won't necessarily completely legalize the operation, but it will 1025 // more effectively move in the right direction and prevent falling down 1026 // to scalarization in many cases due to the input vector being split too 1027 // far. 1028 unsigned NumElements = SrcVT.getVectorNumElements(); 1029 if ((NumElements & 1) == 0 && 1030 SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) { 1031 LLVMContext &Ctx = *DAG.getContext(); 1032 EVT NewSrcVT = EVT::getVectorVT( 1033 Ctx, EVT::getIntegerVT( 1034 Ctx, SrcVT.getVectorElementType().getSizeInBits() * 2), 1035 NumElements); 1036 EVT SplitSrcVT = 1037 EVT::getVectorVT(Ctx, SrcVT.getVectorElementType(), NumElements / 2); 1038 EVT SplitLoVT, SplitHiVT; 1039 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT); 1040 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) && 1041 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) { 1042 DEBUG(dbgs() << "Split vector extend via incremental extend:"; 1043 N->dump(&DAG); dbgs() << "\n"); 1044 // Extend the source vector by one step. 1045 SDValue NewSrc = 1046 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0)); 1047 // Get the low and high halves of the new, extended one step, vector. 1048 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl); 1049 // Extend those vector halves the rest of the way. 1050 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 1051 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 1052 return; 1053 } 1054 } 1055 // Fall back to the generic unary operator splitting otherwise. 1056 SplitVecRes_UnaryOp(N, Lo, Hi); 1057 } 1058 1059 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 1060 SDValue &Lo, SDValue &Hi) { 1061 // The low and high parts of the original input give four input vectors. 1062 SDValue Inputs[4]; 1063 SDLoc dl(N); 1064 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 1065 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 1066 EVT NewVT = Inputs[0].getValueType(); 1067 unsigned NewElts = NewVT.getVectorNumElements(); 1068 1069 // If Lo or Hi uses elements from at most two of the four input vectors, then 1070 // express it as a vector shuffle of those two inputs. Otherwise extract the 1071 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 1072 SmallVector<int, 16> Ops; 1073 for (unsigned High = 0; High < 2; ++High) { 1074 SDValue &Output = High ? Hi : Lo; 1075 1076 // Build a shuffle mask for the output, discovering on the fly which 1077 // input vectors to use as shuffle operands (recorded in InputUsed). 1078 // If building a suitable shuffle vector proves too hard, then bail 1079 // out with useBuildVector set. 1080 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 1081 unsigned FirstMaskIdx = High * NewElts; 1082 bool useBuildVector = false; 1083 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 1084 // The mask element. This indexes into the input. 1085 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 1086 1087 // The input vector this mask element indexes into. 1088 unsigned Input = (unsigned)Idx / NewElts; 1089 1090 if (Input >= array_lengthof(Inputs)) { 1091 // The mask element does not index into any input vector. 1092 Ops.push_back(-1); 1093 continue; 1094 } 1095 1096 // Turn the index into an offset from the start of the input vector. 1097 Idx -= Input * NewElts; 1098 1099 // Find or create a shuffle vector operand to hold this input. 1100 unsigned OpNo; 1101 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 1102 if (InputUsed[OpNo] == Input) { 1103 // This input vector is already an operand. 1104 break; 1105 } else if (InputUsed[OpNo] == -1U) { 1106 // Create a new operand for this input vector. 1107 InputUsed[OpNo] = Input; 1108 break; 1109 } 1110 } 1111 1112 if (OpNo >= array_lengthof(InputUsed)) { 1113 // More than two input vectors used! Give up on trying to create a 1114 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 1115 useBuildVector = true; 1116 break; 1117 } 1118 1119 // Add the mask index for the new shuffle vector. 1120 Ops.push_back(Idx + OpNo * NewElts); 1121 } 1122 1123 if (useBuildVector) { 1124 EVT EltVT = NewVT.getVectorElementType(); 1125 SmallVector<SDValue, 16> SVOps; 1126 1127 // Extract the input elements by hand. 1128 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 1129 // The mask element. This indexes into the input. 1130 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 1131 1132 // The input vector this mask element indexes into. 1133 unsigned Input = (unsigned)Idx / NewElts; 1134 1135 if (Input >= array_lengthof(Inputs)) { 1136 // The mask element is "undef" or indexes off the end of the input. 1137 SVOps.push_back(DAG.getUNDEF(EltVT)); 1138 continue; 1139 } 1140 1141 // Turn the index into an offset from the start of the input vector. 1142 Idx -= Input * NewElts; 1143 1144 // Extract the vector element by hand. 1145 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 1146 Inputs[Input], DAG.getConstant(Idx, 1147 TLI.getVectorIdxTy()))); 1148 } 1149 1150 // Construct the Lo/Hi output using a BUILD_VECTOR. 1151 Output = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, SVOps); 1152 } else if (InputUsed[0] == -1U) { 1153 // No input vectors were used! The result is undefined. 1154 Output = DAG.getUNDEF(NewVT); 1155 } else { 1156 SDValue Op0 = Inputs[InputUsed[0]]; 1157 // If only one input was used, use an undefined vector for the other. 1158 SDValue Op1 = InputUsed[1] == -1U ? 1159 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 1160 // At least one input vector was used. Create a new shuffle vector. 1161 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 1162 } 1163 1164 Ops.clear(); 1165 } 1166 } 1167 1168 1169 //===----------------------------------------------------------------------===// 1170 // Operand Vector Splitting 1171 //===----------------------------------------------------------------------===// 1172 1173 /// SplitVectorOperand - This method is called when the specified operand of the 1174 /// specified node is found to need vector splitting. At this point, all of the 1175 /// result types of the node are known to be legal, but other operands of the 1176 /// node may need legalization as well as the specified one. 1177 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 1178 DEBUG(dbgs() << "Split node operand: "; 1179 N->dump(&DAG); 1180 dbgs() << "\n"); 1181 SDValue Res = SDValue(); 1182 1183 // See if the target wants to custom split this node. 1184 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 1185 return false; 1186 1187 if (!Res.getNode()) { 1188 switch (N->getOpcode()) { 1189 default: 1190 #ifndef NDEBUG 1191 dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 1192 N->dump(&DAG); 1193 dbgs() << "\n"; 1194 #endif 1195 report_fatal_error("Do not know how to split this operator's " 1196 "operand!\n"); 1197 1198 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; 1199 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; 1200 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 1201 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 1202 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; 1203 case ISD::TRUNCATE: Res = SplitVecOp_TRUNCATE(N); break; 1204 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; 1205 case ISD::STORE: 1206 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 1207 break; 1208 case ISD::VSELECT: 1209 Res = SplitVecOp_VSELECT(N, OpNo); 1210 break; 1211 case ISD::CTTZ: 1212 case ISD::CTLZ: 1213 case ISD::CTPOP: 1214 case ISD::FP_EXTEND: 1215 case ISD::FP_TO_SINT: 1216 case ISD::FP_TO_UINT: 1217 case ISD::SINT_TO_FP: 1218 case ISD::UINT_TO_FP: 1219 case ISD::FTRUNC: 1220 case ISD::SIGN_EXTEND: 1221 case ISD::ZERO_EXTEND: 1222 case ISD::ANY_EXTEND: 1223 Res = SplitVecOp_UnaryOp(N); 1224 break; 1225 } 1226 } 1227 1228 // If the result is null, the sub-method took care of registering results etc. 1229 if (!Res.getNode()) return false; 1230 1231 // If the result is N, the sub-method updated N in place. Tell the legalizer 1232 // core about this. 1233 if (Res.getNode() == N) 1234 return true; 1235 1236 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1237 "Invalid operand expansion"); 1238 1239 ReplaceValueWith(SDValue(N, 0), Res); 1240 return false; 1241 } 1242 1243 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { 1244 // The only possibility for an illegal operand is the mask, since result type 1245 // legalization would have handled this node already otherwise. 1246 assert(OpNo == 0 && "Illegal operand must be mask"); 1247 1248 SDValue Mask = N->getOperand(0); 1249 SDValue Src0 = N->getOperand(1); 1250 SDValue Src1 = N->getOperand(2); 1251 EVT Src0VT = Src0.getValueType(); 1252 SDLoc DL(N); 1253 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?"); 1254 1255 SDValue Lo, Hi; 1256 GetSplitVector(N->getOperand(0), Lo, Hi); 1257 assert(Lo.getValueType() == Hi.getValueType() && 1258 "Lo and Hi have differing types"); 1259 1260 EVT LoOpVT, HiOpVT; 1261 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT); 1262 assert(LoOpVT == HiOpVT && "Asymmetric vector split?"); 1263 1264 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask; 1265 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL); 1266 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL); 1267 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL); 1268 1269 SDValue LoSelect = 1270 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1); 1271 SDValue HiSelect = 1272 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1); 1273 1274 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect); 1275 } 1276 1277 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 1278 // The result has a legal vector type, but the input needs splitting. 1279 EVT ResVT = N->getValueType(0); 1280 SDValue Lo, Hi; 1281 SDLoc dl(N); 1282 GetSplitVector(N->getOperand(0), Lo, Hi); 1283 EVT InVT = Lo.getValueType(); 1284 1285 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1286 InVT.getVectorNumElements()); 1287 1288 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 1289 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 1290 1291 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 1292 } 1293 1294 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { 1295 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will 1296 // end up being split all the way down to individual components. Convert the 1297 // split pieces into integers and reassemble. 1298 SDValue Lo, Hi; 1299 GetSplitVector(N->getOperand(0), Lo, Hi); 1300 Lo = BitConvertToInteger(Lo); 1301 Hi = BitConvertToInteger(Hi); 1302 1303 if (TLI.isBigEndian()) 1304 std::swap(Lo, Hi); 1305 1306 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), 1307 JoinIntegers(Lo, Hi)); 1308 } 1309 1310 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1311 // We know that the extracted result type is legal. 1312 EVT SubVT = N->getValueType(0); 1313 SDValue Idx = N->getOperand(1); 1314 SDLoc dl(N); 1315 SDValue Lo, Hi; 1316 GetSplitVector(N->getOperand(0), Lo, Hi); 1317 1318 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1319 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1320 1321 if (IdxVal < LoElts) { 1322 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1323 "Extracted subvector crosses vector split!"); 1324 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1325 } else { 1326 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1327 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1328 } 1329 } 1330 1331 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1332 SDValue Vec = N->getOperand(0); 1333 SDValue Idx = N->getOperand(1); 1334 EVT VecVT = Vec.getValueType(); 1335 1336 if (isa<ConstantSDNode>(Idx)) { 1337 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1338 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1339 1340 SDValue Lo, Hi; 1341 GetSplitVector(Vec, Lo, Hi); 1342 1343 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1344 1345 if (IdxVal < LoElts) 1346 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0); 1347 return SDValue(DAG.UpdateNodeOperands(N, Hi, 1348 DAG.getConstant(IdxVal - LoElts, 1349 Idx.getValueType())), 0); 1350 } 1351 1352 // Store the vector to the stack. 1353 EVT EltVT = VecVT.getVectorElementType(); 1354 SDLoc dl(N); 1355 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1356 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1357 MachinePointerInfo(), false, false, 0); 1358 1359 // Load back the required element. 1360 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1361 return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 1362 MachinePointerInfo(), EltVT, false, false, 0); 1363 } 1364 1365 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1366 assert(N->isUnindexed() && "Indexed store of vector?"); 1367 assert(OpNo == 1 && "Can only split the stored value"); 1368 SDLoc DL(N); 1369 1370 bool isTruncating = N->isTruncatingStore(); 1371 SDValue Ch = N->getChain(); 1372 SDValue Ptr = N->getBasePtr(); 1373 EVT MemoryVT = N->getMemoryVT(); 1374 unsigned Alignment = N->getOriginalAlignment(); 1375 bool isVol = N->isVolatile(); 1376 bool isNT = N->isNonTemporal(); 1377 const MDNode *TBAAInfo = N->getTBAAInfo(); 1378 SDValue Lo, Hi; 1379 GetSplitVector(N->getOperand(1), Lo, Hi); 1380 1381 EVT LoMemVT, HiMemVT; 1382 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1383 1384 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1385 1386 if (isTruncating) 1387 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1388 LoMemVT, isVol, isNT, Alignment, TBAAInfo); 1389 else 1390 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1391 isVol, isNT, Alignment, TBAAInfo); 1392 1393 // Increment the pointer to the other half. 1394 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 1395 DAG.getConstant(IncrementSize, Ptr.getValueType())); 1396 1397 if (isTruncating) 1398 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, 1399 N->getPointerInfo().getWithOffset(IncrementSize), 1400 HiMemVT, isVol, isNT, Alignment, TBAAInfo); 1401 else 1402 Hi = DAG.getStore(Ch, DL, Hi, Ptr, 1403 N->getPointerInfo().getWithOffset(IncrementSize), 1404 isVol, isNT, Alignment, TBAAInfo); 1405 1406 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 1407 } 1408 1409 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { 1410 SDLoc DL(N); 1411 1412 // The input operands all must have the same type, and we know the result 1413 // type is valid. Convert this to a buildvector which extracts all the 1414 // input elements. 1415 // TODO: If the input elements are power-two vectors, we could convert this to 1416 // a new CONCAT_VECTORS node with elements that are half-wide. 1417 SmallVector<SDValue, 32> Elts; 1418 EVT EltVT = N->getValueType(0).getVectorElementType(); 1419 for (unsigned op = 0, e = N->getNumOperands(); op != e; ++op) { 1420 SDValue Op = N->getOperand(op); 1421 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements(); 1422 i != e; ++i) { 1423 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, 1424 Op, DAG.getConstant(i, TLI.getVectorIdxTy()))); 1425 1426 } 1427 } 1428 1429 return DAG.getNode(ISD::BUILD_VECTOR, DL, N->getValueType(0), Elts); 1430 } 1431 1432 SDValue DAGTypeLegalizer::SplitVecOp_TRUNCATE(SDNode *N) { 1433 // The result type is legal, but the input type is illegal. If splitting 1434 // ends up with the result type of each half still being legal, just 1435 // do that. If, however, that would result in an illegal result type, 1436 // we can try to get more clever with power-two vectors. Specifically, 1437 // split the input type, but also widen the result element size, then 1438 // concatenate the halves and truncate again. For example, consider a target 1439 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit 1440 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do: 1441 // %inlo = v4i32 extract_subvector %in, 0 1442 // %inhi = v4i32 extract_subvector %in, 4 1443 // %lo16 = v4i16 trunc v4i32 %inlo 1444 // %hi16 = v4i16 trunc v4i32 %inhi 1445 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16 1446 // %res = v8i8 trunc v8i16 %in16 1447 // 1448 // Without this transform, the original truncate would end up being 1449 // scalarized, which is pretty much always a last resort. 1450 SDValue InVec = N->getOperand(0); 1451 EVT InVT = InVec->getValueType(0); 1452 EVT OutVT = N->getValueType(0); 1453 unsigned NumElements = OutVT.getVectorNumElements(); 1454 // Widening should have already made sure this is a power-two vector 1455 // if we're trying to split it at all. assert() that's true, just in case. 1456 assert(!(NumElements & 1) && "Splitting vector, but not in half!"); 1457 1458 unsigned InElementSize = InVT.getVectorElementType().getSizeInBits(); 1459 unsigned OutElementSize = OutVT.getVectorElementType().getSizeInBits(); 1460 1461 // If the input elements are only 1/2 the width of the result elements, 1462 // just use the normal splitting. Our trick only work if there's room 1463 // to split more than once. 1464 if (InElementSize <= OutElementSize * 2) 1465 return SplitVecOp_UnaryOp(N); 1466 SDLoc DL(N); 1467 1468 // Extract the halves of the input via extract_subvector. 1469 SDValue InLoVec, InHiVec; 1470 std::tie(InLoVec, InHiVec) = DAG.SplitVector(InVec, DL); 1471 // Truncate them to 1/2 the element size. 1472 EVT HalfElementVT = EVT::getIntegerVT(*DAG.getContext(), InElementSize/2); 1473 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, 1474 NumElements/2); 1475 SDValue HalfLo = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, InLoVec); 1476 SDValue HalfHi = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, InHiVec); 1477 // Concatenate them to get the full intermediate truncation result. 1478 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements); 1479 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo, 1480 HalfHi); 1481 // Now finish up by truncating all the way down to the original result 1482 // type. This should normally be something that ends up being legal directly, 1483 // but in theory if a target has very wide vectors and an annoyingly 1484 // restricted set of legal types, this split can chain to build things up. 1485 return DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec); 1486 } 1487 1488 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { 1489 assert(N->getValueType(0).isVector() && 1490 N->getOperand(0).getValueType().isVector() && 1491 "Operand types must be vectors"); 1492 // The result has a legal vector type, but the input needs splitting. 1493 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; 1494 SDLoc DL(N); 1495 GetSplitVector(N->getOperand(0), Lo0, Hi0); 1496 GetSplitVector(N->getOperand(1), Lo1, Hi1); 1497 unsigned PartElements = Lo0.getValueType().getVectorNumElements(); 1498 EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements); 1499 EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements); 1500 1501 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2)); 1502 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2)); 1503 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes); 1504 return PromoteTargetBoolean(Con, N->getValueType(0)); 1505 } 1506 1507 1508 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { 1509 // The result has a legal vector type, but the input needs splitting. 1510 EVT ResVT = N->getValueType(0); 1511 SDValue Lo, Hi; 1512 SDLoc DL(N); 1513 GetSplitVector(N->getOperand(0), Lo, Hi); 1514 EVT InVT = Lo.getValueType(); 1515 1516 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1517 InVT.getVectorNumElements()); 1518 1519 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); 1520 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); 1521 1522 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); 1523 } 1524 1525 1526 1527 //===----------------------------------------------------------------------===// 1528 // Result Vector Widening 1529 //===----------------------------------------------------------------------===// 1530 1531 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1532 DEBUG(dbgs() << "Widen node result " << ResNo << ": "; 1533 N->dump(&DAG); 1534 dbgs() << "\n"); 1535 1536 // See if the target wants to custom widen this node. 1537 if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 1538 return; 1539 1540 SDValue Res = SDValue(); 1541 switch (N->getOpcode()) { 1542 default: 1543 #ifndef NDEBUG 1544 dbgs() << "WidenVectorResult #" << ResNo << ": "; 1545 N->dump(&DAG); 1546 dbgs() << "\n"; 1547 #endif 1548 llvm_unreachable("Do not know how to widen the result of this operator!"); 1549 1550 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; 1551 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; 1552 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1553 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1554 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1555 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1556 case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break; 1557 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1558 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1559 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1560 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 1561 case ISD::VSELECT: 1562 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1563 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1564 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break; 1565 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1566 case ISD::VECTOR_SHUFFLE: 1567 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 1568 break; 1569 1570 case ISD::ADD: 1571 case ISD::AND: 1572 case ISD::MUL: 1573 case ISD::MULHS: 1574 case ISD::MULHU: 1575 case ISD::OR: 1576 case ISD::SUB: 1577 case ISD::XOR: 1578 Res = WidenVecRes_Binary(N); 1579 break; 1580 1581 case ISD::FADD: 1582 case ISD::FCOPYSIGN: 1583 case ISD::FMUL: 1584 case ISD::FPOW: 1585 case ISD::FSUB: 1586 case ISD::FDIV: 1587 case ISD::FREM: 1588 case ISD::SDIV: 1589 case ISD::UDIV: 1590 case ISD::SREM: 1591 case ISD::UREM: 1592 Res = WidenVecRes_BinaryCanTrap(N); 1593 break; 1594 1595 case ISD::FPOWI: 1596 Res = WidenVecRes_POWI(N); 1597 break; 1598 1599 case ISD::SHL: 1600 case ISD::SRA: 1601 case ISD::SRL: 1602 Res = WidenVecRes_Shift(N); 1603 break; 1604 1605 case ISD::ANY_EXTEND: 1606 case ISD::FP_EXTEND: 1607 case ISD::FP_ROUND: 1608 case ISD::FP_TO_SINT: 1609 case ISD::FP_TO_UINT: 1610 case ISD::SIGN_EXTEND: 1611 case ISD::SINT_TO_FP: 1612 case ISD::TRUNCATE: 1613 case ISD::UINT_TO_FP: 1614 case ISD::ZERO_EXTEND: 1615 Res = WidenVecRes_Convert(N); 1616 break; 1617 1618 case ISD::BSWAP: 1619 case ISD::CTLZ: 1620 case ISD::CTPOP: 1621 case ISD::CTTZ: 1622 case ISD::FABS: 1623 case ISD::FCEIL: 1624 case ISD::FCOS: 1625 case ISD::FEXP: 1626 case ISD::FEXP2: 1627 case ISD::FFLOOR: 1628 case ISD::FLOG: 1629 case ISD::FLOG10: 1630 case ISD::FLOG2: 1631 case ISD::FNEARBYINT: 1632 case ISD::FNEG: 1633 case ISD::FRINT: 1634 case ISD::FROUND: 1635 case ISD::FSIN: 1636 case ISD::FSQRT: 1637 case ISD::FTRUNC: 1638 Res = WidenVecRes_Unary(N); 1639 break; 1640 case ISD::FMA: 1641 Res = WidenVecRes_Ternary(N); 1642 break; 1643 } 1644 1645 // If Res is null, the sub-method took care of registering the result. 1646 if (Res.getNode()) 1647 SetWidenedVector(SDValue(N, ResNo), Res); 1648 } 1649 1650 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { 1651 // Ternary op widening. 1652 SDLoc dl(N); 1653 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1654 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1655 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1656 SDValue InOp3 = GetWidenedVector(N->getOperand(2)); 1657 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3); 1658 } 1659 1660 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1661 // Binary op widening. 1662 SDLoc dl(N); 1663 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1664 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1665 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1666 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2); 1667 } 1668 1669 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) { 1670 // Binary op widening for operations that can trap. 1671 unsigned Opcode = N->getOpcode(); 1672 SDLoc dl(N); 1673 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1674 EVT WidenEltVT = WidenVT.getVectorElementType(); 1675 EVT VT = WidenVT; 1676 unsigned NumElts = VT.getVectorNumElements(); 1677 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 1678 NumElts = NumElts / 2; 1679 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1680 } 1681 1682 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) { 1683 // Operation doesn't trap so just widen as normal. 1684 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1685 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1686 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2); 1687 } 1688 1689 // No legal vector version so unroll the vector operation and then widen. 1690 if (NumElts == 1) 1691 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 1692 1693 // Since the operation can trap, apply operation on the original vector. 1694 EVT MaxVT = VT; 1695 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1696 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1697 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 1698 1699 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 1700 unsigned ConcatEnd = 0; // Current ConcatOps index. 1701 int Idx = 0; // Current Idx into input vectors. 1702 1703 // NumElts := greatest legal vector size (at most WidenVT) 1704 // while (orig. vector has unhandled elements) { 1705 // take munches of size NumElts from the beginning and add to ConcatOps 1706 // NumElts := next smaller supported vector size or 1 1707 // } 1708 while (CurNumElts != 0) { 1709 while (CurNumElts >= NumElts) { 1710 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1, 1711 DAG.getConstant(Idx, TLI.getVectorIdxTy())); 1712 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2, 1713 DAG.getConstant(Idx, TLI.getVectorIdxTy())); 1714 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2); 1715 Idx += NumElts; 1716 CurNumElts -= NumElts; 1717 } 1718 do { 1719 NumElts = NumElts / 2; 1720 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1721 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 1722 1723 if (NumElts == 1) { 1724 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 1725 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1726 InOp1, DAG.getConstant(Idx, 1727 TLI.getVectorIdxTy())); 1728 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1729 InOp2, DAG.getConstant(Idx, 1730 TLI.getVectorIdxTy())); 1731 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT, 1732 EOp1, EOp2); 1733 } 1734 CurNumElts = 0; 1735 } 1736 } 1737 1738 // Check to see if we have a single operation with the widen type. 1739 if (ConcatEnd == 1) { 1740 VT = ConcatOps[0].getValueType(); 1741 if (VT == WidenVT) 1742 return ConcatOps[0]; 1743 } 1744 1745 // while (Some element of ConcatOps is not of type MaxVT) { 1746 // From the end of ConcatOps, collect elements of the same type and put 1747 // them into an op of the next larger supported type 1748 // } 1749 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { 1750 Idx = ConcatEnd - 1; 1751 VT = ConcatOps[Idx--].getValueType(); 1752 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) 1753 Idx--; 1754 1755 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1; 1756 EVT NextVT; 1757 do { 1758 NextSize *= 2; 1759 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize); 1760 } while (!TLI.isTypeLegal(NextVT)); 1761 1762 if (!VT.isVector()) { 1763 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT 1764 SDValue VecOp = DAG.getUNDEF(NextVT); 1765 unsigned NumToInsert = ConcatEnd - Idx - 1; 1766 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) { 1767 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, 1768 ConcatOps[OpIdx], DAG.getConstant(i, 1769 TLI.getVectorIdxTy())); 1770 } 1771 ConcatOps[Idx+1] = VecOp; 1772 ConcatEnd = Idx + 2; 1773 } else { 1774 // Vector type, create a CONCAT_VECTORS of type NextVT 1775 SDValue undefVec = DAG.getUNDEF(VT); 1776 unsigned OpsToConcat = NextSize/VT.getVectorNumElements(); 1777 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat); 1778 unsigned RealVals = ConcatEnd - Idx - 1; 1779 unsigned SubConcatEnd = 0; 1780 unsigned SubConcatIdx = Idx + 1; 1781 while (SubConcatEnd < RealVals) 1782 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx]; 1783 while (SubConcatEnd < OpsToConcat) 1784 SubConcatOps[SubConcatEnd++] = undefVec; 1785 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1786 NextVT, SubConcatOps); 1787 ConcatEnd = SubConcatIdx + 1; 1788 } 1789 } 1790 1791 // Check to see if we have a single operation with the widen type. 1792 if (ConcatEnd == 1) { 1793 VT = ConcatOps[0].getValueType(); 1794 if (VT == WidenVT) 1795 return ConcatOps[0]; 1796 } 1797 1798 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT 1799 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements(); 1800 if (NumOps != ConcatEnd ) { 1801 SDValue UndefVal = DAG.getUNDEF(MaxVT); 1802 for (unsigned j = ConcatEnd; j < NumOps; ++j) 1803 ConcatOps[j] = UndefVal; 1804 } 1805 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 1806 makeArrayRef(ConcatOps.data(), NumOps)); 1807 } 1808 1809 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1810 SDValue InOp = N->getOperand(0); 1811 SDLoc DL(N); 1812 1813 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1814 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1815 1816 EVT InVT = InOp.getValueType(); 1817 EVT InEltVT = InVT.getVectorElementType(); 1818 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1819 1820 unsigned Opcode = N->getOpcode(); 1821 unsigned InVTNumElts = InVT.getVectorNumElements(); 1822 1823 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 1824 InOp = GetWidenedVector(N->getOperand(0)); 1825 InVT = InOp.getValueType(); 1826 InVTNumElts = InVT.getVectorNumElements(); 1827 if (InVTNumElts == WidenNumElts) { 1828 if (N->getNumOperands() == 1) 1829 return DAG.getNode(Opcode, DL, WidenVT, InOp); 1830 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1)); 1831 } 1832 } 1833 1834 if (TLI.isTypeLegal(InWidenVT)) { 1835 // Because the result and the input are different vector types, widening 1836 // the result could create a legal type but widening the input might make 1837 // it an illegal type that might lead to repeatedly splitting the input 1838 // and then widening it. To avoid this, we widen the input only if 1839 // it results in a legal type. 1840 if (WidenNumElts % InVTNumElts == 0) { 1841 // Widen the input and call convert on the widened input vector. 1842 unsigned NumConcat = WidenNumElts/InVTNumElts; 1843 SmallVector<SDValue, 16> Ops(NumConcat); 1844 Ops[0] = InOp; 1845 SDValue UndefVal = DAG.getUNDEF(InVT); 1846 for (unsigned i = 1; i != NumConcat; ++i) 1847 Ops[i] = UndefVal; 1848 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops); 1849 if (N->getNumOperands() == 1) 1850 return DAG.getNode(Opcode, DL, WidenVT, InVec); 1851 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1)); 1852 } 1853 1854 if (InVTNumElts % WidenNumElts == 0) { 1855 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, 1856 InOp, DAG.getConstant(0, 1857 TLI.getVectorIdxTy())); 1858 // Extract the input and convert the shorten input vector. 1859 if (N->getNumOperands() == 1) 1860 return DAG.getNode(Opcode, DL, WidenVT, InVal); 1861 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1)); 1862 } 1863 } 1864 1865 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1866 SmallVector<SDValue, 16> Ops(WidenNumElts); 1867 EVT EltVT = WidenVT.getVectorElementType(); 1868 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1869 unsigned i; 1870 for (i=0; i < MinElts; ++i) { 1871 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 1872 DAG.getConstant(i, TLI.getVectorIdxTy())); 1873 if (N->getNumOperands() == 1) 1874 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val); 1875 else 1876 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1)); 1877 } 1878 1879 SDValue UndefVal = DAG.getUNDEF(EltVT); 1880 for (; i < WidenNumElts; ++i) 1881 Ops[i] = UndefVal; 1882 1883 return DAG.getNode(ISD::BUILD_VECTOR, DL, WidenVT, Ops); 1884 } 1885 1886 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) { 1887 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1888 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1889 SDValue ShOp = N->getOperand(1); 1890 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 1891 } 1892 1893 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 1894 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1895 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1896 SDValue ShOp = N->getOperand(1); 1897 1898 EVT ShVT = ShOp.getValueType(); 1899 if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) { 1900 ShOp = GetWidenedVector(ShOp); 1901 ShVT = ShOp.getValueType(); 1902 } 1903 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), 1904 ShVT.getVectorElementType(), 1905 WidenVT.getVectorNumElements()); 1906 if (ShVT != ShWidenVT) 1907 ShOp = ModifyToType(ShOp, ShWidenVT); 1908 1909 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 1910 } 1911 1912 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 1913 // Unary op widening. 1914 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1915 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1916 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp); 1917 } 1918 1919 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 1920 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1921 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 1922 cast<VTSDNode>(N->getOperand(1))->getVT() 1923 .getVectorElementType(), 1924 WidenVT.getVectorNumElements()); 1925 SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 1926 return DAG.getNode(N->getOpcode(), SDLoc(N), 1927 WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 1928 } 1929 1930 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { 1931 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); 1932 return GetWidenedVector(WidenVec); 1933 } 1934 1935 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { 1936 SDValue InOp = N->getOperand(0); 1937 EVT InVT = InOp.getValueType(); 1938 EVT VT = N->getValueType(0); 1939 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1940 SDLoc dl(N); 1941 1942 switch (getTypeAction(InVT)) { 1943 case TargetLowering::TypeLegal: 1944 break; 1945 case TargetLowering::TypePromoteInteger: 1946 // If the incoming type is a vector that is being promoted, then 1947 // we know that the elements are arranged differently and that we 1948 // must perform the conversion using a stack slot. 1949 if (InVT.isVector()) 1950 break; 1951 1952 // If the InOp is promoted to the same size, convert it. Otherwise, 1953 // fall out of the switch and widen the promoted input. 1954 InOp = GetPromotedInteger(InOp); 1955 InVT = InOp.getValueType(); 1956 if (WidenVT.bitsEq(InVT)) 1957 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 1958 break; 1959 case TargetLowering::TypeSoftenFloat: 1960 case TargetLowering::TypeExpandInteger: 1961 case TargetLowering::TypeExpandFloat: 1962 case TargetLowering::TypeScalarizeVector: 1963 case TargetLowering::TypeSplitVector: 1964 break; 1965 case TargetLowering::TypeWidenVector: 1966 // If the InOp is widened to the same size, convert it. Otherwise, fall 1967 // out of the switch and widen the widened input. 1968 InOp = GetWidenedVector(InOp); 1969 InVT = InOp.getValueType(); 1970 if (WidenVT.bitsEq(InVT)) 1971 // The input widens to the same size. Convert to the widen value. 1972 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 1973 break; 1974 } 1975 1976 unsigned WidenSize = WidenVT.getSizeInBits(); 1977 unsigned InSize = InVT.getSizeInBits(); 1978 // x86mmx is not an acceptable vector element type, so don't try. 1979 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) { 1980 // Determine new input vector type. The new input vector type will use 1981 // the same element type (if its a vector) or use the input type as a 1982 // vector. It is the same size as the type to widen to. 1983 EVT NewInVT; 1984 unsigned NewNumElts = WidenSize / InSize; 1985 if (InVT.isVector()) { 1986 EVT InEltVT = InVT.getVectorElementType(); 1987 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, 1988 WidenSize / InEltVT.getSizeInBits()); 1989 } else { 1990 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 1991 } 1992 1993 if (TLI.isTypeLegal(NewInVT)) { 1994 // Because the result and the input are different vector types, widening 1995 // the result could create a legal type but widening the input might make 1996 // it an illegal type that might lead to repeatedly splitting the input 1997 // and then widening it. To avoid this, we widen the input only if 1998 // it results in a legal type. 1999 SmallVector<SDValue, 16> Ops(NewNumElts); 2000 SDValue UndefVal = DAG.getUNDEF(InVT); 2001 Ops[0] = InOp; 2002 for (unsigned i = 1; i < NewNumElts; ++i) 2003 Ops[i] = UndefVal; 2004 2005 SDValue NewVec; 2006 if (InVT.isVector()) 2007 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops); 2008 else 2009 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, NewInVT, Ops); 2010 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec); 2011 } 2012 } 2013 2014 return CreateStackStoreLoad(InOp, WidenVT); 2015 } 2016 2017 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 2018 SDLoc dl(N); 2019 // Build a vector with undefined for the new nodes. 2020 EVT VT = N->getValueType(0); 2021 2022 // Integer BUILD_VECTOR operands may be larger than the node's vector element 2023 // type. The UNDEFs need to have the same type as the existing operands. 2024 EVT EltVT = N->getOperand(0).getValueType(); 2025 unsigned NumElts = VT.getVectorNumElements(); 2026 2027 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2028 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2029 2030 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 2031 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!"); 2032 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT)); 2033 2034 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, NewOps); 2035 } 2036 2037 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 2038 EVT InVT = N->getOperand(0).getValueType(); 2039 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2040 SDLoc dl(N); 2041 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2042 unsigned NumInElts = InVT.getVectorNumElements(); 2043 unsigned NumOperands = N->getNumOperands(); 2044 2045 bool InputWidened = false; // Indicates we need to widen the input. 2046 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) { 2047 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 2048 // Add undef vectors to widen to correct length. 2049 unsigned NumConcat = WidenVT.getVectorNumElements() / 2050 InVT.getVectorNumElements(); 2051 SDValue UndefVal = DAG.getUNDEF(InVT); 2052 SmallVector<SDValue, 16> Ops(NumConcat); 2053 for (unsigned i=0; i < NumOperands; ++i) 2054 Ops[i] = N->getOperand(i); 2055 for (unsigned i = NumOperands; i != NumConcat; ++i) 2056 Ops[i] = UndefVal; 2057 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops); 2058 } 2059 } else { 2060 InputWidened = true; 2061 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 2062 // The inputs and the result are widen to the same value. 2063 unsigned i; 2064 for (i=1; i < NumOperands; ++i) 2065 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 2066 break; 2067 2068 if (i == NumOperands) 2069 // Everything but the first operand is an UNDEF so just return the 2070 // widened first operand. 2071 return GetWidenedVector(N->getOperand(0)); 2072 2073 if (NumOperands == 2) { 2074 // Replace concat of two operands with a shuffle. 2075 SmallVector<int, 16> MaskOps(WidenNumElts, -1); 2076 for (unsigned i = 0; i < NumInElts; ++i) { 2077 MaskOps[i] = i; 2078 MaskOps[i + NumInElts] = i + WidenNumElts; 2079 } 2080 return DAG.getVectorShuffle(WidenVT, dl, 2081 GetWidenedVector(N->getOperand(0)), 2082 GetWidenedVector(N->getOperand(1)), 2083 &MaskOps[0]); 2084 } 2085 } 2086 } 2087 2088 // Fall back to use extracts and build vector. 2089 EVT EltVT = WidenVT.getVectorElementType(); 2090 SmallVector<SDValue, 16> Ops(WidenNumElts); 2091 unsigned Idx = 0; 2092 for (unsigned i=0; i < NumOperands; ++i) { 2093 SDValue InOp = N->getOperand(i); 2094 if (InputWidened) 2095 InOp = GetWidenedVector(InOp); 2096 for (unsigned j=0; j < NumInElts; ++j) 2097 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2098 DAG.getConstant(j, TLI.getVectorIdxTy())); 2099 } 2100 SDValue UndefVal = DAG.getUNDEF(EltVT); 2101 for (; Idx < WidenNumElts; ++Idx) 2102 Ops[Idx] = UndefVal; 2103 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2104 } 2105 2106 SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 2107 SDLoc dl(N); 2108 SDValue InOp = N->getOperand(0); 2109 SDValue RndOp = N->getOperand(3); 2110 SDValue SatOp = N->getOperand(4); 2111 2112 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2113 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2114 2115 EVT InVT = InOp.getValueType(); 2116 EVT InEltVT = InVT.getVectorElementType(); 2117 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 2118 2119 SDValue DTyOp = DAG.getValueType(WidenVT); 2120 SDValue STyOp = DAG.getValueType(InWidenVT); 2121 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 2122 2123 unsigned InVTNumElts = InVT.getVectorNumElements(); 2124 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 2125 InOp = GetWidenedVector(InOp); 2126 InVT = InOp.getValueType(); 2127 InVTNumElts = InVT.getVectorNumElements(); 2128 if (InVTNumElts == WidenNumElts) 2129 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2130 SatOp, CvtCode); 2131 } 2132 2133 if (TLI.isTypeLegal(InWidenVT)) { 2134 // Because the result and the input are different vector types, widening 2135 // the result could create a legal type but widening the input might make 2136 // it an illegal type that might lead to repeatedly splitting the input 2137 // and then widening it. To avoid this, we widen the input only if 2138 // it results in a legal type. 2139 if (WidenNumElts % InVTNumElts == 0) { 2140 // Widen the input and call convert on the widened input vector. 2141 unsigned NumConcat = WidenNumElts/InVTNumElts; 2142 SmallVector<SDValue, 16> Ops(NumConcat); 2143 Ops[0] = InOp; 2144 SDValue UndefVal = DAG.getUNDEF(InVT); 2145 for (unsigned i = 1; i != NumConcat; ++i) 2146 Ops[i] = UndefVal; 2147 2148 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, Ops); 2149 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2150 SatOp, CvtCode); 2151 } 2152 2153 if (InVTNumElts % WidenNumElts == 0) { 2154 // Extract the input and convert the shorten input vector. 2155 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 2156 DAG.getConstant(0, TLI.getVectorIdxTy())); 2157 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2158 SatOp, CvtCode); 2159 } 2160 } 2161 2162 // Otherwise unroll into some nasty scalar code and rebuild the vector. 2163 SmallVector<SDValue, 16> Ops(WidenNumElts); 2164 EVT EltVT = WidenVT.getVectorElementType(); 2165 DTyOp = DAG.getValueType(EltVT); 2166 STyOp = DAG.getValueType(InEltVT); 2167 2168 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 2169 unsigned i; 2170 for (i=0; i < MinElts; ++i) { 2171 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 2172 DAG.getConstant(i, TLI.getVectorIdxTy())); 2173 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 2174 SatOp, CvtCode); 2175 } 2176 2177 SDValue UndefVal = DAG.getUNDEF(EltVT); 2178 for (; i < WidenNumElts; ++i) 2179 Ops[i] = UndefVal; 2180 2181 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2182 } 2183 2184 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 2185 EVT VT = N->getValueType(0); 2186 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2187 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2188 SDValue InOp = N->getOperand(0); 2189 SDValue Idx = N->getOperand(1); 2190 SDLoc dl(N); 2191 2192 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2193 InOp = GetWidenedVector(InOp); 2194 2195 EVT InVT = InOp.getValueType(); 2196 2197 // Check if we can just return the input vector after widening. 2198 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2199 if (IdxVal == 0 && InVT == WidenVT) 2200 return InOp; 2201 2202 // Check if we can extract from the vector. 2203 unsigned InNumElts = InVT.getVectorNumElements(); 2204 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 2205 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 2206 2207 // We could try widening the input to the right length but for now, extract 2208 // the original elements, fill the rest with undefs and build a vector. 2209 SmallVector<SDValue, 16> Ops(WidenNumElts); 2210 EVT EltVT = VT.getVectorElementType(); 2211 unsigned NumElts = VT.getVectorNumElements(); 2212 unsigned i; 2213 for (i=0; i < NumElts; ++i) 2214 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2215 DAG.getConstant(IdxVal+i, TLI.getVectorIdxTy())); 2216 2217 SDValue UndefVal = DAG.getUNDEF(EltVT); 2218 for (; i < WidenNumElts; ++i) 2219 Ops[i] = UndefVal; 2220 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2221 } 2222 2223 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 2224 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2225 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), 2226 InOp.getValueType(), InOp, 2227 N->getOperand(1), N->getOperand(2)); 2228 } 2229 2230 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 2231 LoadSDNode *LD = cast<LoadSDNode>(N); 2232 ISD::LoadExtType ExtType = LD->getExtensionType(); 2233 2234 SDValue Result; 2235 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 2236 if (ExtType != ISD::NON_EXTLOAD) 2237 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); 2238 else 2239 Result = GenWidenVectorLoads(LdChain, LD); 2240 2241 // If we generate a single load, we can use that for the chain. Otherwise, 2242 // build a factor node to remember the multiple loads are independent and 2243 // chain to that. 2244 SDValue NewChain; 2245 if (LdChain.size() == 1) 2246 NewChain = LdChain[0]; 2247 else 2248 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain); 2249 2250 // Modified the chain - switch anything that used the old chain to use 2251 // the new one. 2252 ReplaceValueWith(SDValue(N, 1), NewChain); 2253 2254 return Result; 2255 } 2256 2257 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 2258 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2259 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), 2260 WidenVT, N->getOperand(0)); 2261 } 2262 2263 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 2264 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2265 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2266 2267 SDValue Cond1 = N->getOperand(0); 2268 EVT CondVT = Cond1.getValueType(); 2269 if (CondVT.isVector()) { 2270 EVT CondEltVT = CondVT.getVectorElementType(); 2271 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), 2272 CondEltVT, WidenNumElts); 2273 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector) 2274 Cond1 = GetWidenedVector(Cond1); 2275 2276 // If we have to split the condition there is no point in widening the 2277 // select. This would result in an cycle of widening the select -> 2278 // widening the condition operand -> splitting the condition operand -> 2279 // splitting the select -> widening the select. Instead split this select 2280 // further and widen the resulting type. 2281 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) { 2282 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0); 2283 SDValue Res = ModifyToType(SplitSelect, WidenVT); 2284 return Res; 2285 } 2286 2287 if (Cond1.getValueType() != CondWidenVT) 2288 Cond1 = ModifyToType(Cond1, CondWidenVT); 2289 } 2290 2291 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 2292 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 2293 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 2294 return DAG.getNode(N->getOpcode(), SDLoc(N), 2295 WidenVT, Cond1, InOp1, InOp2); 2296 } 2297 2298 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 2299 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 2300 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 2301 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 2302 InOp1.getValueType(), N->getOperand(0), 2303 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 2304 } 2305 2306 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { 2307 assert(N->getValueType(0).isVector() == 2308 N->getOperand(0).getValueType().isVector() && 2309 "Scalar/Vector type mismatch"); 2310 if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N); 2311 2312 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2313 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2314 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2315 return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT, 2316 InOp1, InOp2, N->getOperand(2)); 2317 } 2318 2319 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 2320 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2321 return DAG.getUNDEF(WidenVT); 2322 } 2323 2324 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 2325 EVT VT = N->getValueType(0); 2326 SDLoc dl(N); 2327 2328 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2329 unsigned NumElts = VT.getVectorNumElements(); 2330 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2331 2332 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2333 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2334 2335 // Adjust mask based on new input vector length. 2336 SmallVector<int, 16> NewMask; 2337 for (unsigned i = 0; i != NumElts; ++i) { 2338 int Idx = N->getMaskElt(i); 2339 if (Idx < (int)NumElts) 2340 NewMask.push_back(Idx); 2341 else 2342 NewMask.push_back(Idx - NumElts + WidenNumElts); 2343 } 2344 for (unsigned i = NumElts; i != WidenNumElts; ++i) 2345 NewMask.push_back(-1); 2346 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 2347 } 2348 2349 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 2350 assert(N->getValueType(0).isVector() && 2351 N->getOperand(0).getValueType().isVector() && 2352 "Operands must be vectors"); 2353 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2354 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2355 2356 SDValue InOp1 = N->getOperand(0); 2357 EVT InVT = InOp1.getValueType(); 2358 assert(InVT.isVector() && "can not widen non-vector type"); 2359 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), 2360 InVT.getVectorElementType(), WidenNumElts); 2361 InOp1 = GetWidenedVector(InOp1); 2362 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2363 2364 // Assume that the input and output will be widen appropriately. If not, 2365 // we will have to unroll it at some point. 2366 assert(InOp1.getValueType() == WidenInVT && 2367 InOp2.getValueType() == WidenInVT && 2368 "Input not widened to expected type!"); 2369 (void)WidenInVT; 2370 return DAG.getNode(ISD::SETCC, SDLoc(N), 2371 WidenVT, InOp1, InOp2, N->getOperand(2)); 2372 } 2373 2374 2375 //===----------------------------------------------------------------------===// 2376 // Widen Vector Operand 2377 //===----------------------------------------------------------------------===// 2378 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { 2379 DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; 2380 N->dump(&DAG); 2381 dbgs() << "\n"); 2382 SDValue Res = SDValue(); 2383 2384 // See if the target wants to custom widen this node. 2385 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2386 return false; 2387 2388 switch (N->getOpcode()) { 2389 default: 2390 #ifndef NDEBUG 2391 dbgs() << "WidenVectorOperand op #" << OpNo << ": "; 2392 N->dump(&DAG); 2393 dbgs() << "\n"; 2394 #endif 2395 llvm_unreachable("Do not know how to widen this operator's operand!"); 2396 2397 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break; 2398 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 2399 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 2400 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 2401 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 2402 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; 2403 2404 case ISD::ANY_EXTEND: 2405 case ISD::SIGN_EXTEND: 2406 case ISD::ZERO_EXTEND: 2407 Res = WidenVecOp_EXTEND(N); 2408 break; 2409 2410 case ISD::FP_EXTEND: 2411 case ISD::FP_TO_SINT: 2412 case ISD::FP_TO_UINT: 2413 case ISD::SINT_TO_FP: 2414 case ISD::UINT_TO_FP: 2415 case ISD::TRUNCATE: 2416 Res = WidenVecOp_Convert(N); 2417 break; 2418 } 2419 2420 // If Res is null, the sub-method took care of registering the result. 2421 if (!Res.getNode()) return false; 2422 2423 // If the result is N, the sub-method updated N in place. Tell the legalizer 2424 // core about this. 2425 if (Res.getNode() == N) 2426 return true; 2427 2428 2429 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2430 "Invalid operand expansion"); 2431 2432 ReplaceValueWith(SDValue(N, 0), Res); 2433 return false; 2434 } 2435 2436 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) { 2437 SDLoc DL(N); 2438 EVT VT = N->getValueType(0); 2439 2440 SDValue InOp = N->getOperand(0); 2441 // If some legalization strategy other than widening is used on the operand, 2442 // we can't safely assume that just extending the low lanes is the correct 2443 // transformation. 2444 if (getTypeAction(InOp.getValueType()) != TargetLowering::TypeWidenVector) 2445 return WidenVecOp_Convert(N); 2446 InOp = GetWidenedVector(InOp); 2447 assert(VT.getVectorNumElements() < 2448 InOp.getValueType().getVectorNumElements() && 2449 "Input wasn't widened!"); 2450 2451 // We may need to further widen the operand until it has the same total 2452 // vector size as the result. 2453 EVT InVT = InOp.getValueType(); 2454 if (InVT.getSizeInBits() != VT.getSizeInBits()) { 2455 EVT InEltVT = InVT.getVectorElementType(); 2456 for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) { 2457 EVT FixedVT = (MVT::SimpleValueType)i; 2458 EVT FixedEltVT = FixedVT.getVectorElementType(); 2459 if (TLI.isTypeLegal(FixedVT) && 2460 FixedVT.getSizeInBits() == VT.getSizeInBits() && 2461 FixedEltVT == InEltVT) { 2462 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() && 2463 "Not enough elements in the fixed type for the operand!"); 2464 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() && 2465 "We can't have the same type as we started with!"); 2466 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements()) 2467 InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT, 2468 DAG.getUNDEF(FixedVT), InOp, 2469 DAG.getConstant(0, TLI.getVectorIdxTy())); 2470 else 2471 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp, 2472 DAG.getConstant(0, TLI.getVectorIdxTy())); 2473 break; 2474 } 2475 } 2476 InVT = InOp.getValueType(); 2477 if (InVT.getSizeInBits() != VT.getSizeInBits()) 2478 // We couldn't find a legal vector type that was a widening of the input 2479 // and could be extended in-register to the result type, so we have to 2480 // scalarize. 2481 return WidenVecOp_Convert(N); 2482 } 2483 2484 // Use special DAG nodes to represent the operation of extending the 2485 // low lanes. 2486 switch (N->getOpcode()) { 2487 default: 2488 llvm_unreachable("Extend legalization on on extend operation!"); 2489 case ISD::ANY_EXTEND: 2490 return DAG.getAnyExtendVectorInReg(InOp, DL, VT); 2491 case ISD::SIGN_EXTEND: 2492 return DAG.getSignExtendVectorInReg(InOp, DL, VT); 2493 case ISD::ZERO_EXTEND: 2494 return DAG.getZeroExtendVectorInReg(InOp, DL, VT); 2495 } 2496 } 2497 2498 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 2499 // Since the result is legal and the input is illegal, it is unlikely 2500 // that we can fix the input to a legal type so unroll the convert 2501 // into some scalar code and create a nasty build vector. 2502 EVT VT = N->getValueType(0); 2503 EVT EltVT = VT.getVectorElementType(); 2504 SDLoc dl(N); 2505 unsigned NumElts = VT.getVectorNumElements(); 2506 SDValue InOp = N->getOperand(0); 2507 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2508 InOp = GetWidenedVector(InOp); 2509 EVT InVT = InOp.getValueType(); 2510 EVT InEltVT = InVT.getVectorElementType(); 2511 2512 unsigned Opcode = N->getOpcode(); 2513 SmallVector<SDValue, 16> Ops(NumElts); 2514 for (unsigned i=0; i < NumElts; ++i) 2515 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 2516 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 2517 DAG.getConstant(i, TLI.getVectorIdxTy()))); 2518 2519 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); 2520 } 2521 2522 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { 2523 EVT VT = N->getValueType(0); 2524 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2525 EVT InWidenVT = InOp.getValueType(); 2526 SDLoc dl(N); 2527 2528 // Check if we can convert between two legal vector types and extract. 2529 unsigned InWidenSize = InWidenVT.getSizeInBits(); 2530 unsigned Size = VT.getSizeInBits(); 2531 // x86mmx is not an acceptable vector element type, so don't try. 2532 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) { 2533 unsigned NewNumElts = InWidenSize / Size; 2534 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 2535 if (TLI.isTypeLegal(NewVT)) { 2536 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 2537 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 2538 DAG.getConstant(0, TLI.getVectorIdxTy())); 2539 } 2540 } 2541 2542 return CreateStackStoreLoad(InOp, VT); 2543 } 2544 2545 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 2546 // If the input vector is not legal, it is likely that we will not find a 2547 // legal vector of the same size. Replace the concatenate vector with a 2548 // nasty build vector. 2549 EVT VT = N->getValueType(0); 2550 EVT EltVT = VT.getVectorElementType(); 2551 SDLoc dl(N); 2552 unsigned NumElts = VT.getVectorNumElements(); 2553 SmallVector<SDValue, 16> Ops(NumElts); 2554 2555 EVT InVT = N->getOperand(0).getValueType(); 2556 unsigned NumInElts = InVT.getVectorNumElements(); 2557 2558 unsigned Idx = 0; 2559 unsigned NumOperands = N->getNumOperands(); 2560 for (unsigned i=0; i < NumOperands; ++i) { 2561 SDValue InOp = N->getOperand(i); 2562 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2563 InOp = GetWidenedVector(InOp); 2564 for (unsigned j=0; j < NumInElts; ++j) 2565 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2566 DAG.getConstant(j, TLI.getVectorIdxTy())); 2567 } 2568 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); 2569 } 2570 2571 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 2572 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2573 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), 2574 N->getValueType(0), InOp, N->getOperand(1)); 2575 } 2576 2577 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 2578 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2579 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 2580 N->getValueType(0), InOp, N->getOperand(1)); 2581 } 2582 2583 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 2584 // We have to widen the value but we want only to store the original 2585 // vector type. 2586 StoreSDNode *ST = cast<StoreSDNode>(N); 2587 2588 SmallVector<SDValue, 16> StChain; 2589 if (ST->isTruncatingStore()) 2590 GenWidenVectorTruncStores(StChain, ST); 2591 else 2592 GenWidenVectorStores(StChain, ST); 2593 2594 if (StChain.size() == 1) 2595 return StChain[0]; 2596 else 2597 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain); 2598 } 2599 2600 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { 2601 SDValue InOp0 = GetWidenedVector(N->getOperand(0)); 2602 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 2603 SDLoc dl(N); 2604 2605 // WARNING: In this code we widen the compare instruction with garbage. 2606 // This garbage may contain denormal floats which may be slow. Is this a real 2607 // concern ? Should we zero the unused lanes if this is a float compare ? 2608 2609 // Get a new SETCC node to compare the newly widened operands. 2610 // Only some of the compared elements are legal. 2611 EVT SVT = TLI.getSetCCResultType(*DAG.getContext(), InOp0.getValueType()); 2612 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N), 2613 SVT, InOp0, InOp1, N->getOperand(2)); 2614 2615 // Extract the needed results from the result vector. 2616 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), 2617 SVT.getVectorElementType(), 2618 N->getValueType(0).getVectorNumElements()); 2619 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, 2620 ResVT, WideSETCC, DAG.getConstant(0, 2621 TLI.getVectorIdxTy())); 2622 2623 return PromoteTargetBoolean(CC, N->getValueType(0)); 2624 } 2625 2626 2627 //===----------------------------------------------------------------------===// 2628 // Vector Widening Utilities 2629 //===----------------------------------------------------------------------===// 2630 2631 // Utility function to find the type to chop up a widen vector for load/store 2632 // TLI: Target lowering used to determine legal types. 2633 // Width: Width left need to load/store. 2634 // WidenVT: The widen vector type to load to/store from 2635 // Align: If 0, don't allow use of a wider type 2636 // WidenEx: If Align is not 0, the amount additional we can load/store from. 2637 2638 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI, 2639 unsigned Width, EVT WidenVT, 2640 unsigned Align = 0, unsigned WidenEx = 0) { 2641 EVT WidenEltVT = WidenVT.getVectorElementType(); 2642 unsigned WidenWidth = WidenVT.getSizeInBits(); 2643 unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); 2644 unsigned AlignInBits = Align*8; 2645 2646 // If we have one element to load/store, return it. 2647 EVT RetVT = WidenEltVT; 2648 if (Width == WidenEltWidth) 2649 return RetVT; 2650 2651 // See if there is larger legal integer than the element type to load/store 2652 unsigned VT; 2653 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE; 2654 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) { 2655 EVT MemVT((MVT::SimpleValueType) VT); 2656 unsigned MemVTWidth = MemVT.getSizeInBits(); 2657 if (MemVT.getSizeInBits() <= WidenEltWidth) 2658 break; 2659 if (TLI.isTypeLegal(MemVT) && (WidenWidth % MemVTWidth) == 0 && 2660 isPowerOf2_32(WidenWidth / MemVTWidth) && 2661 (MemVTWidth <= Width || 2662 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2663 RetVT = MemVT; 2664 break; 2665 } 2666 } 2667 2668 // See if there is a larger vector type to load/store that has the same vector 2669 // element type and is evenly divisible with the WidenVT. 2670 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE; 2671 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) { 2672 EVT MemVT = (MVT::SimpleValueType) VT; 2673 unsigned MemVTWidth = MemVT.getSizeInBits(); 2674 if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() && 2675 (WidenWidth % MemVTWidth) == 0 && 2676 isPowerOf2_32(WidenWidth / MemVTWidth) && 2677 (MemVTWidth <= Width || 2678 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2679 if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT) 2680 return MemVT; 2681 } 2682 } 2683 2684 return RetVT; 2685 } 2686 2687 // Builds a vector type from scalar loads 2688 // VecTy: Resulting Vector type 2689 // LDOps: Load operators to build a vector type 2690 // [Start,End) the list of loads to use. 2691 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, 2692 SmallVectorImpl<SDValue> &LdOps, 2693 unsigned Start, unsigned End) { 2694 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2695 SDLoc dl(LdOps[Start]); 2696 EVT LdTy = LdOps[Start].getValueType(); 2697 unsigned Width = VecTy.getSizeInBits(); 2698 unsigned NumElts = Width / LdTy.getSizeInBits(); 2699 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts); 2700 2701 unsigned Idx = 1; 2702 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]); 2703 2704 for (unsigned i = Start + 1; i != End; ++i) { 2705 EVT NewLdTy = LdOps[i].getValueType(); 2706 if (NewLdTy != LdTy) { 2707 NumElts = Width / NewLdTy.getSizeInBits(); 2708 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts); 2709 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp); 2710 // Readjust position and vector position based on new load type 2711 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); 2712 LdTy = NewLdTy; 2713 } 2714 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i], 2715 DAG.getConstant(Idx++, TLI.getVectorIdxTy())); 2716 } 2717 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp); 2718 } 2719 2720 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain, 2721 LoadSDNode *LD) { 2722 // The strategy assumes that we can efficiently load powers of two widths. 2723 // The routines chops the vector into the largest vector loads with the same 2724 // element type or scalar loads and then recombines it to the widen vector 2725 // type. 2726 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2727 unsigned WidenWidth = WidenVT.getSizeInBits(); 2728 EVT LdVT = LD->getMemoryVT(); 2729 SDLoc dl(LD); 2730 assert(LdVT.isVector() && WidenVT.isVector()); 2731 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 2732 2733 // Load information 2734 SDValue Chain = LD->getChain(); 2735 SDValue BasePtr = LD->getBasePtr(); 2736 unsigned Align = LD->getAlignment(); 2737 bool isVolatile = LD->isVolatile(); 2738 bool isNonTemporal = LD->isNonTemporal(); 2739 bool isInvariant = LD->isInvariant(); 2740 const MDNode *TBAAInfo = LD->getTBAAInfo(); 2741 2742 int LdWidth = LdVT.getSizeInBits(); 2743 int WidthDiff = WidenWidth - LdWidth; // Difference 2744 unsigned LdAlign = (isVolatile) ? 0 : Align; // Allow wider loads 2745 2746 // Find the vector type that can load from. 2747 EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2748 int NewVTWidth = NewVT.getSizeInBits(); 2749 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(), 2750 isVolatile, isNonTemporal, isInvariant, Align, 2751 TBAAInfo); 2752 LdChain.push_back(LdOp.getValue(1)); 2753 2754 // Check if we can load the element with one instruction 2755 if (LdWidth <= NewVTWidth) { 2756 if (!NewVT.isVector()) { 2757 unsigned NumElts = WidenWidth / NewVTWidth; 2758 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2759 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2760 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp); 2761 } 2762 if (NewVT == WidenVT) 2763 return LdOp; 2764 2765 assert(WidenWidth % NewVTWidth == 0); 2766 unsigned NumConcat = WidenWidth / NewVTWidth; 2767 SmallVector<SDValue, 16> ConcatOps(NumConcat); 2768 SDValue UndefVal = DAG.getUNDEF(NewVT); 2769 ConcatOps[0] = LdOp; 2770 for (unsigned i = 1; i != NumConcat; ++i) 2771 ConcatOps[i] = UndefVal; 2772 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps); 2773 } 2774 2775 // Load vector by using multiple loads from largest vector to scalar 2776 SmallVector<SDValue, 16> LdOps; 2777 LdOps.push_back(LdOp); 2778 2779 LdWidth -= NewVTWidth; 2780 unsigned Offset = 0; 2781 2782 while (LdWidth > 0) { 2783 unsigned Increment = NewVTWidth / 8; 2784 Offset += Increment; 2785 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2786 DAG.getConstant(Increment, BasePtr.getValueType())); 2787 2788 SDValue L; 2789 if (LdWidth < NewVTWidth) { 2790 // Our current type we are using is too large, find a better size 2791 NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2792 NewVTWidth = NewVT.getSizeInBits(); 2793 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 2794 LD->getPointerInfo().getWithOffset(Offset), isVolatile, 2795 isNonTemporal, isInvariant, MinAlign(Align, Increment), 2796 TBAAInfo); 2797 LdChain.push_back(L.getValue(1)); 2798 if (L->getValueType(0).isVector()) { 2799 SmallVector<SDValue, 16> Loads; 2800 Loads.push_back(L); 2801 unsigned size = L->getValueSizeInBits(0); 2802 while (size < LdOp->getValueSizeInBits(0)) { 2803 Loads.push_back(DAG.getUNDEF(L->getValueType(0))); 2804 size += L->getValueSizeInBits(0); 2805 } 2806 L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads); 2807 } 2808 } else { 2809 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 2810 LD->getPointerInfo().getWithOffset(Offset), isVolatile, 2811 isNonTemporal, isInvariant, MinAlign(Align, Increment), 2812 TBAAInfo); 2813 LdChain.push_back(L.getValue(1)); 2814 } 2815 2816 LdOps.push_back(L); 2817 2818 2819 LdWidth -= NewVTWidth; 2820 } 2821 2822 // Build the vector from the loads operations 2823 unsigned End = LdOps.size(); 2824 if (!LdOps[0].getValueType().isVector()) 2825 // All the loads are scalar loads. 2826 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End); 2827 2828 // If the load contains vectors, build the vector using concat vector. 2829 // All of the vectors used to loads are power of 2 and the scalars load 2830 // can be combined to make a power of 2 vector. 2831 SmallVector<SDValue, 16> ConcatOps(End); 2832 int i = End - 1; 2833 int Idx = End; 2834 EVT LdTy = LdOps[i].getValueType(); 2835 // First combine the scalar loads to a vector 2836 if (!LdTy.isVector()) { 2837 for (--i; i >= 0; --i) { 2838 LdTy = LdOps[i].getValueType(); 2839 if (LdTy.isVector()) 2840 break; 2841 } 2842 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i+1, End); 2843 } 2844 ConcatOps[--Idx] = LdOps[i]; 2845 for (--i; i >= 0; --i) { 2846 EVT NewLdTy = LdOps[i].getValueType(); 2847 if (NewLdTy != LdTy) { 2848 // Create a larger vector 2849 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy, 2850 makeArrayRef(&ConcatOps[Idx], End - Idx)); 2851 Idx = End - 1; 2852 LdTy = NewLdTy; 2853 } 2854 ConcatOps[--Idx] = LdOps[i]; 2855 } 2856 2857 if (WidenWidth == LdTy.getSizeInBits()*(End - Idx)) 2858 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 2859 makeArrayRef(&ConcatOps[Idx], End - Idx)); 2860 2861 // We need to fill the rest with undefs to build the vector 2862 unsigned NumOps = WidenWidth / LdTy.getSizeInBits(); 2863 SmallVector<SDValue, 16> WidenOps(NumOps); 2864 SDValue UndefVal = DAG.getUNDEF(LdTy); 2865 { 2866 unsigned i = 0; 2867 for (; i != End-Idx; ++i) 2868 WidenOps[i] = ConcatOps[Idx+i]; 2869 for (; i != NumOps; ++i) 2870 WidenOps[i] = UndefVal; 2871 } 2872 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps); 2873 } 2874 2875 SDValue 2876 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain, 2877 LoadSDNode *LD, 2878 ISD::LoadExtType ExtType) { 2879 // For extension loads, it may not be more efficient to chop up the vector 2880 // and then extended it. Instead, we unroll the load and build a new vector. 2881 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2882 EVT LdVT = LD->getMemoryVT(); 2883 SDLoc dl(LD); 2884 assert(LdVT.isVector() && WidenVT.isVector()); 2885 2886 // Load information 2887 SDValue Chain = LD->getChain(); 2888 SDValue BasePtr = LD->getBasePtr(); 2889 unsigned Align = LD->getAlignment(); 2890 bool isVolatile = LD->isVolatile(); 2891 bool isNonTemporal = LD->isNonTemporal(); 2892 const MDNode *TBAAInfo = LD->getTBAAInfo(); 2893 2894 EVT EltVT = WidenVT.getVectorElementType(); 2895 EVT LdEltVT = LdVT.getVectorElementType(); 2896 unsigned NumElts = LdVT.getVectorNumElements(); 2897 2898 // Load each element and widen 2899 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2900 SmallVector<SDValue, 16> Ops(WidenNumElts); 2901 unsigned Increment = LdEltVT.getSizeInBits() / 8; 2902 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, 2903 LD->getPointerInfo(), 2904 LdEltVT, isVolatile, isNonTemporal, Align, TBAAInfo); 2905 LdChain.push_back(Ops[0].getValue(1)); 2906 unsigned i = 0, Offset = Increment; 2907 for (i=1; i < NumElts; ++i, Offset += Increment) { 2908 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 2909 BasePtr, 2910 DAG.getConstant(Offset, 2911 BasePtr.getValueType())); 2912 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, 2913 LD->getPointerInfo().getWithOffset(Offset), LdEltVT, 2914 isVolatile, isNonTemporal, Align, TBAAInfo); 2915 LdChain.push_back(Ops[i].getValue(1)); 2916 } 2917 2918 // Fill the rest with undefs 2919 SDValue UndefVal = DAG.getUNDEF(EltVT); 2920 for (; i != WidenNumElts; ++i) 2921 Ops[i] = UndefVal; 2922 2923 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2924 } 2925 2926 2927 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, 2928 StoreSDNode *ST) { 2929 // The strategy assumes that we can efficiently store powers of two widths. 2930 // The routines chops the vector into the largest vector stores with the same 2931 // element type or scalar stores. 2932 SDValue Chain = ST->getChain(); 2933 SDValue BasePtr = ST->getBasePtr(); 2934 unsigned Align = ST->getAlignment(); 2935 bool isVolatile = ST->isVolatile(); 2936 bool isNonTemporal = ST->isNonTemporal(); 2937 const MDNode *TBAAInfo = ST->getTBAAInfo(); 2938 SDValue ValOp = GetWidenedVector(ST->getValue()); 2939 SDLoc dl(ST); 2940 2941 EVT StVT = ST->getMemoryVT(); 2942 unsigned StWidth = StVT.getSizeInBits(); 2943 EVT ValVT = ValOp.getValueType(); 2944 unsigned ValWidth = ValVT.getSizeInBits(); 2945 EVT ValEltVT = ValVT.getVectorElementType(); 2946 unsigned ValEltWidth = ValEltVT.getSizeInBits(); 2947 assert(StVT.getVectorElementType() == ValEltVT); 2948 2949 int Idx = 0; // current index to store 2950 unsigned Offset = 0; // offset from base to store 2951 while (StWidth != 0) { 2952 // Find the largest vector type we can store with 2953 EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT); 2954 unsigned NewVTWidth = NewVT.getSizeInBits(); 2955 unsigned Increment = NewVTWidth / 8; 2956 if (NewVT.isVector()) { 2957 unsigned NumVTElts = NewVT.getVectorNumElements(); 2958 do { 2959 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp, 2960 DAG.getConstant(Idx, TLI.getVectorIdxTy())); 2961 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 2962 ST->getPointerInfo().getWithOffset(Offset), 2963 isVolatile, isNonTemporal, 2964 MinAlign(Align, Offset), TBAAInfo)); 2965 StWidth -= NewVTWidth; 2966 Offset += Increment; 2967 Idx += NumVTElts; 2968 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2969 DAG.getConstant(Increment, BasePtr.getValueType())); 2970 } while (StWidth != 0 && StWidth >= NewVTWidth); 2971 } else { 2972 // Cast the vector to the scalar type we can store 2973 unsigned NumElts = ValWidth / NewVTWidth; 2974 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2975 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp); 2976 // Readjust index position based on new vector type 2977 Idx = Idx * ValEltWidth / NewVTWidth; 2978 do { 2979 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp, 2980 DAG.getConstant(Idx++, TLI.getVectorIdxTy())); 2981 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 2982 ST->getPointerInfo().getWithOffset(Offset), 2983 isVolatile, isNonTemporal, 2984 MinAlign(Align, Offset), TBAAInfo)); 2985 StWidth -= NewVTWidth; 2986 Offset += Increment; 2987 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2988 DAG.getConstant(Increment, BasePtr.getValueType())); 2989 } while (StWidth != 0 && StWidth >= NewVTWidth); 2990 // Restore index back to be relative to the original widen element type 2991 Idx = Idx * NewVTWidth / ValEltWidth; 2992 } 2993 } 2994 } 2995 2996 void 2997 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain, 2998 StoreSDNode *ST) { 2999 // For extension loads, it may not be more efficient to truncate the vector 3000 // and then store it. Instead, we extract each element and then store it. 3001 SDValue Chain = ST->getChain(); 3002 SDValue BasePtr = ST->getBasePtr(); 3003 unsigned Align = ST->getAlignment(); 3004 bool isVolatile = ST->isVolatile(); 3005 bool isNonTemporal = ST->isNonTemporal(); 3006 const MDNode *TBAAInfo = ST->getTBAAInfo(); 3007 SDValue ValOp = GetWidenedVector(ST->getValue()); 3008 SDLoc dl(ST); 3009 3010 EVT StVT = ST->getMemoryVT(); 3011 EVT ValVT = ValOp.getValueType(); 3012 3013 // It must be true that we the widen vector type is bigger than where 3014 // we need to store. 3015 assert(StVT.isVector() && ValOp.getValueType().isVector()); 3016 assert(StVT.bitsLT(ValOp.getValueType())); 3017 3018 // For truncating stores, we can not play the tricks of chopping legal 3019 // vector types and bit cast it to the right type. Instead, we unroll 3020 // the store. 3021 EVT StEltVT = StVT.getVectorElementType(); 3022 EVT ValEltVT = ValVT.getVectorElementType(); 3023 unsigned Increment = ValEltVT.getSizeInBits() / 8; 3024 unsigned NumElts = StVT.getVectorNumElements(); 3025 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 3026 DAG.getConstant(0, TLI.getVectorIdxTy())); 3027 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, 3028 ST->getPointerInfo(), StEltVT, 3029 isVolatile, isNonTemporal, Align, 3030 TBAAInfo)); 3031 unsigned Offset = Increment; 3032 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 3033 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 3034 BasePtr, DAG.getConstant(Offset, 3035 BasePtr.getValueType())); 3036 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 3037 DAG.getConstant(0, TLI.getVectorIdxTy())); 3038 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, 3039 ST->getPointerInfo().getWithOffset(Offset), 3040 StEltVT, isVolatile, isNonTemporal, 3041 MinAlign(Align, Offset), TBAAInfo)); 3042 } 3043 } 3044 3045 /// Modifies a vector input (widen or narrows) to a vector of NVT. The 3046 /// input vector must have the same element type as NVT. 3047 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) { 3048 // Note that InOp might have been widened so it might already have 3049 // the right width or it might need be narrowed. 3050 EVT InVT = InOp.getValueType(); 3051 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 3052 "input and widen element type must match"); 3053 SDLoc dl(InOp); 3054 3055 // Check if InOp already has the right width. 3056 if (InVT == NVT) 3057 return InOp; 3058 3059 unsigned InNumElts = InVT.getVectorNumElements(); 3060 unsigned WidenNumElts = NVT.getVectorNumElements(); 3061 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 3062 unsigned NumConcat = WidenNumElts / InNumElts; 3063 SmallVector<SDValue, 16> Ops(NumConcat); 3064 SDValue UndefVal = DAG.getUNDEF(InVT); 3065 Ops[0] = InOp; 3066 for (unsigned i = 1; i != NumConcat; ++i) 3067 Ops[i] = UndefVal; 3068 3069 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops); 3070 } 3071 3072 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 3073 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 3074 DAG.getConstant(0, TLI.getVectorIdxTy())); 3075 3076 // Fall back to extract and build. 3077 SmallVector<SDValue, 16> Ops(WidenNumElts); 3078 EVT EltVT = NVT.getVectorElementType(); 3079 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 3080 unsigned Idx; 3081 for (Idx = 0; Idx < MinNumElts; ++Idx) 3082 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 3083 DAG.getConstant(Idx, TLI.getVectorIdxTy())); 3084 3085 SDValue UndefVal = DAG.getUNDEF(EltVT); 3086 for ( ; Idx < WidenNumElts; ++Idx) 3087 Ops[Idx] = UndefVal; 3088 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, Ops); 3089 } 3090