1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file performs vector type splitting and scalarization for LegalizeTypes. 11 // Scalarization is the act of changing a computation in an illegal one-element 12 // vector type to be a computation in its scalar element type. For example, 13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which 15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16 // types. 17 // Splitting is the act of changing a computation in an invalid vector type to 18 // be a computation in two vectors of half the size. For example, implementing 19 // <128 x f32> operations in terms of two <64 x f32> operations. 20 // 21 //===----------------------------------------------------------------------===// 22 23 #include "LegalizeTypes.h" 24 #include "llvm/IR/DataLayout.h" 25 #include "llvm/Support/ErrorHandling.h" 26 #include "llvm/Support/raw_ostream.h" 27 using namespace llvm; 28 29 #define DEBUG_TYPE "legalize-types" 30 31 //===----------------------------------------------------------------------===// 32 // Result Vector Scalarization: <1 x ty> -> ty. 33 //===----------------------------------------------------------------------===// 34 35 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 36 DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; 37 N->dump(&DAG); 38 dbgs() << "\n"); 39 SDValue R = SDValue(); 40 41 switch (N->getOpcode()) { 42 default: 43 #ifndef NDEBUG 44 dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 45 N->dump(&DAG); 46 dbgs() << "\n"; 47 #endif 48 report_fatal_error("Do not know how to scalarize the result of this " 49 "operator!\n"); 50 51 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; 52 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; 53 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; 54 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 55 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 56 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; 57 case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break; 58 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 59 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 60 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 61 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 62 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 63 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break; 64 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 65 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 66 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 67 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 68 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 69 case ISD::ANY_EXTEND: 70 case ISD::BSWAP: 71 case ISD::CTLZ: 72 case ISD::CTLZ_ZERO_UNDEF: 73 case ISD::CTPOP: 74 case ISD::CTTZ: 75 case ISD::CTTZ_ZERO_UNDEF: 76 case ISD::FABS: 77 case ISD::FCEIL: 78 case ISD::FCOS: 79 case ISD::FEXP: 80 case ISD::FEXP2: 81 case ISD::FFLOOR: 82 case ISD::FLOG: 83 case ISD::FLOG10: 84 case ISD::FLOG2: 85 case ISD::FNEARBYINT: 86 case ISD::FNEG: 87 case ISD::FP_EXTEND: 88 case ISD::FP_TO_SINT: 89 case ISD::FP_TO_UINT: 90 case ISD::FRINT: 91 case ISD::FROUND: 92 case ISD::FSIN: 93 case ISD::FSQRT: 94 case ISD::FTRUNC: 95 case ISD::SIGN_EXTEND: 96 case ISD::SINT_TO_FP: 97 case ISD::TRUNCATE: 98 case ISD::UINT_TO_FP: 99 case ISD::ZERO_EXTEND: 100 R = ScalarizeVecRes_UnaryOp(N); 101 break; 102 103 case ISD::ADD: 104 case ISD::AND: 105 case ISD::FADD: 106 case ISD::FCOPYSIGN: 107 case ISD::FDIV: 108 case ISD::FMUL: 109 case ISD::FMINNUM: 110 case ISD::FMAXNUM: 111 112 case ISD::FPOW: 113 case ISD::FREM: 114 case ISD::FSUB: 115 case ISD::MUL: 116 case ISD::OR: 117 case ISD::SDIV: 118 case ISD::SREM: 119 case ISD::SUB: 120 case ISD::UDIV: 121 case ISD::UREM: 122 case ISD::XOR: 123 case ISD::SHL: 124 case ISD::SRA: 125 case ISD::SRL: 126 R = ScalarizeVecRes_BinOp(N); 127 break; 128 case ISD::FMA: 129 R = ScalarizeVecRes_TernaryOp(N); 130 break; 131 } 132 133 // If R is null, the sub-method took care of registering the result. 134 if (R.getNode()) 135 SetScalarizedVector(SDValue(N, ResNo), R); 136 } 137 138 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 139 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 140 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 141 return DAG.getNode(N->getOpcode(), SDLoc(N), 142 LHS.getValueType(), LHS, RHS); 143 } 144 145 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { 146 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 147 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 148 SDValue Op2 = GetScalarizedVector(N->getOperand(2)); 149 return DAG.getNode(N->getOpcode(), SDLoc(N), 150 Op0.getValueType(), Op0, Op1, Op2); 151 } 152 153 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, 154 unsigned ResNo) { 155 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 156 return GetScalarizedVector(Op); 157 } 158 159 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { 160 EVT NewVT = N->getValueType(0).getVectorElementType(); 161 return DAG.getNode(ISD::BITCAST, SDLoc(N), 162 NewVT, N->getOperand(0)); 163 } 164 165 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { 166 EVT EltVT = N->getValueType(0).getVectorElementType(); 167 SDValue InOp = N->getOperand(0); 168 // The BUILD_VECTOR operands may be of wider element types and 169 // we may need to truncate them back to the requested return type. 170 if (EltVT.isInteger()) 171 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 172 return InOp; 173 } 174 175 SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 176 EVT NewVT = N->getValueType(0).getVectorElementType(); 177 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 178 return DAG.getConvertRndSat(NewVT, SDLoc(N), 179 Op0, DAG.getValueType(NewVT), 180 DAG.getValueType(Op0.getValueType()), 181 N->getOperand(3), 182 N->getOperand(4), 183 cast<CvtRndSatSDNode>(N)->getCvtCode()); 184 } 185 186 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 187 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 188 N->getValueType(0).getVectorElementType(), 189 N->getOperand(0), N->getOperand(1)); 190 } 191 192 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { 193 EVT NewVT = N->getValueType(0).getVectorElementType(); 194 SDValue Op = GetScalarizedVector(N->getOperand(0)); 195 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 196 NewVT, Op, N->getOperand(1)); 197 } 198 199 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 200 SDValue Op = GetScalarizedVector(N->getOperand(0)); 201 return DAG.getNode(ISD::FPOWI, SDLoc(N), 202 Op.getValueType(), Op, N->getOperand(1)); 203 } 204 205 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 206 // The value to insert may have a wider type than the vector element type, 207 // so be sure to truncate it to the element type if necessary. 208 SDValue Op = N->getOperand(1); 209 EVT EltVT = N->getValueType(0).getVectorElementType(); 210 if (Op.getValueType() != EltVT) 211 // FIXME: Can this happen for floating point types? 212 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op); 213 return Op; 214 } 215 216 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 217 assert(N->isUnindexed() && "Indexed vector load?"); 218 219 SDValue Result = DAG.getLoad(ISD::UNINDEXED, 220 N->getExtensionType(), 221 N->getValueType(0).getVectorElementType(), 222 SDLoc(N), 223 N->getChain(), N->getBasePtr(), 224 DAG.getUNDEF(N->getBasePtr().getValueType()), 225 N->getPointerInfo(), 226 N->getMemoryVT().getVectorElementType(), 227 N->isVolatile(), N->isNonTemporal(), 228 N->isInvariant(), N->getOriginalAlignment(), 229 N->getAAInfo()); 230 231 // Legalized the chain result - switch anything that used the old chain to 232 // use the new one. 233 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 234 return Result; 235 } 236 237 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 238 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 239 EVT DestVT = N->getValueType(0).getVectorElementType(); 240 SDValue Op = N->getOperand(0); 241 EVT OpVT = Op.getValueType(); 242 SDLoc DL(N); 243 // The result needs scalarizing, but it's not a given that the source does. 244 // This is a workaround for targets where it's impossible to scalarize the 245 // result of a conversion, because the source type is legal. 246 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32} 247 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is 248 // legal and was not scalarized. 249 // See the similar logic in ScalarizeVecRes_VSETCC 250 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 251 Op = GetScalarizedVector(Op); 252 } else { 253 EVT VT = OpVT.getVectorElementType(); 254 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, 255 DAG.getConstant(0, TLI.getVectorIdxTy())); 256 } 257 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op); 258 } 259 260 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 261 EVT EltVT = N->getValueType(0).getVectorElementType(); 262 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 263 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 264 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT, 265 LHS, DAG.getValueType(ExtVT)); 266 } 267 268 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 269 // If the operand is wider than the vector element type then it is implicitly 270 // truncated. Make that explicit here. 271 EVT EltVT = N->getValueType(0).getVectorElementType(); 272 SDValue InOp = N->getOperand(0); 273 if (InOp.getValueType() != EltVT) 274 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 275 return InOp; 276 } 277 278 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { 279 SDValue Cond = GetScalarizedVector(N->getOperand(0)); 280 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 281 TargetLowering::BooleanContent ScalarBool = 282 TLI.getBooleanContents(false, false); 283 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false); 284 285 // If integer and float booleans have different contents then we can't 286 // reliably optimize in all cases. There is a full explanation for this in 287 // DAGCombiner::visitSELECT() where the same issue affects folding 288 // (select C, 0, 1) to (xor C, 1). 289 if (TLI.getBooleanContents(false, false) != 290 TLI.getBooleanContents(false, true)) { 291 // At least try the common case where the boolean is generated by a 292 // comparison. 293 if (Cond->getOpcode() == ISD::SETCC) { 294 EVT OpVT = Cond->getOperand(0)->getValueType(0); 295 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType()); 296 VecBool = TLI.getBooleanContents(OpVT); 297 } else 298 ScalarBool = TargetLowering::UndefinedBooleanContent; 299 } 300 301 if (ScalarBool != VecBool) { 302 EVT CondVT = Cond.getValueType(); 303 switch (ScalarBool) { 304 case TargetLowering::UndefinedBooleanContent: 305 break; 306 case TargetLowering::ZeroOrOneBooleanContent: 307 assert(VecBool == TargetLowering::UndefinedBooleanContent || 308 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent); 309 // Vector read from all ones, scalar expects a single 1 so mask. 310 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT, 311 Cond, DAG.getConstant(1, CondVT)); 312 break; 313 case TargetLowering::ZeroOrNegativeOneBooleanContent: 314 assert(VecBool == TargetLowering::UndefinedBooleanContent || 315 VecBool == TargetLowering::ZeroOrOneBooleanContent); 316 // Vector reads from a one, scalar from all ones so sign extend. 317 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT, 318 Cond, DAG.getValueType(MVT::i1)); 319 break; 320 } 321 } 322 323 return DAG.getSelect(SDLoc(N), 324 LHS.getValueType(), Cond, LHS, 325 GetScalarizedVector(N->getOperand(2))); 326 } 327 328 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 329 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 330 return DAG.getSelect(SDLoc(N), 331 LHS.getValueType(), N->getOperand(0), LHS, 332 GetScalarizedVector(N->getOperand(2))); 333 } 334 335 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 336 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 337 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(), 338 N->getOperand(0), N->getOperand(1), 339 LHS, GetScalarizedVector(N->getOperand(3)), 340 N->getOperand(4)); 341 } 342 343 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 344 assert(N->getValueType(0).isVector() == 345 N->getOperand(0).getValueType().isVector() && 346 "Scalar/Vector type mismatch"); 347 348 if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N); 349 350 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 351 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 352 SDLoc DL(N); 353 354 // Turn it into a scalar SETCC. 355 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 356 } 357 358 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 359 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 360 } 361 362 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 363 // Figure out if the scalar is the LHS or RHS and return it. 364 SDValue Arg = N->getOperand(2).getOperand(0); 365 if (Arg.getOpcode() == ISD::UNDEF) 366 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 367 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 368 return GetScalarizedVector(N->getOperand(Op)); 369 } 370 371 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 372 assert(N->getValueType(0).isVector() && 373 N->getOperand(0).getValueType().isVector() && 374 "Operand types must be vectors"); 375 SDValue LHS = N->getOperand(0); 376 SDValue RHS = N->getOperand(1); 377 EVT OpVT = LHS.getValueType(); 378 EVT NVT = N->getValueType(0).getVectorElementType(); 379 SDLoc DL(N); 380 381 // The result needs scalarizing, but it's not a given that the source does. 382 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 383 LHS = GetScalarizedVector(LHS); 384 RHS = GetScalarizedVector(RHS); 385 } else { 386 EVT VT = OpVT.getVectorElementType(); 387 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS, 388 DAG.getConstant(0, TLI.getVectorIdxTy())); 389 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS, 390 DAG.getConstant(0, TLI.getVectorIdxTy())); 391 } 392 393 // Turn it into a scalar SETCC. 394 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 395 N->getOperand(2)); 396 // Vectors may have a different boolean contents to scalars. Promote the 397 // value appropriately. 398 ISD::NodeType ExtendCode = 399 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 400 return DAG.getNode(ExtendCode, DL, NVT, Res); 401 } 402 403 404 //===----------------------------------------------------------------------===// 405 // Operand Vector Scalarization <1 x ty> -> ty. 406 //===----------------------------------------------------------------------===// 407 408 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 409 DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; 410 N->dump(&DAG); 411 dbgs() << "\n"); 412 SDValue Res = SDValue(); 413 414 if (!Res.getNode()) { 415 switch (N->getOpcode()) { 416 default: 417 #ifndef NDEBUG 418 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 419 N->dump(&DAG); 420 dbgs() << "\n"; 421 #endif 422 llvm_unreachable("Do not know how to scalarize this operator's operand!"); 423 case ISD::BITCAST: 424 Res = ScalarizeVecOp_BITCAST(N); 425 break; 426 case ISD::ANY_EXTEND: 427 case ISD::ZERO_EXTEND: 428 case ISD::SIGN_EXTEND: 429 case ISD::TRUNCATE: 430 case ISD::FP_TO_SINT: 431 case ISD::FP_TO_UINT: 432 case ISD::SINT_TO_FP: 433 case ISD::UINT_TO_FP: 434 Res = ScalarizeVecOp_UnaryOp(N); 435 break; 436 case ISD::CONCAT_VECTORS: 437 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 438 break; 439 case ISD::EXTRACT_VECTOR_ELT: 440 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 441 break; 442 case ISD::VSELECT: 443 Res = ScalarizeVecOp_VSELECT(N); 444 break; 445 case ISD::STORE: 446 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 447 break; 448 case ISD::FP_ROUND: 449 Res = ScalarizeVecOp_FP_ROUND(N, OpNo); 450 break; 451 } 452 } 453 454 // If the result is null, the sub-method took care of registering results etc. 455 if (!Res.getNode()) return false; 456 457 // If the result is N, the sub-method updated N in place. Tell the legalizer 458 // core about this. 459 if (Res.getNode() == N) 460 return true; 461 462 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 463 "Invalid operand expansion"); 464 465 ReplaceValueWith(SDValue(N, 0), Res); 466 return false; 467 } 468 469 /// ScalarizeVecOp_BITCAST - If the value to convert is a vector that needs 470 /// to be scalarized, it must be <1 x ty>. Convert the element instead. 471 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { 472 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 473 return DAG.getNode(ISD::BITCAST, SDLoc(N), 474 N->getValueType(0), Elt); 475 } 476 477 /// ScalarizeVecOp_UnaryOp - If the input is a vector that needs to be 478 /// scalarized, it must be <1 x ty>. Do the operation on the element instead. 479 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) { 480 assert(N->getValueType(0).getVectorNumElements() == 1 && 481 "Unexpected vector type!"); 482 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 483 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N), 484 N->getValueType(0).getScalarType(), Elt); 485 // Revectorize the result so the types line up with what the uses of this 486 // expression expect. 487 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), Op); 488 } 489 490 /// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 491 /// use a BUILD_VECTOR instead. 492 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 493 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 494 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 495 Ops[i] = GetScalarizedVector(N->getOperand(i)); 496 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), Ops); 497 } 498 499 /// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 500 /// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 501 /// index. 502 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 503 SDValue Res = GetScalarizedVector(N->getOperand(0)); 504 if (Res.getValueType() != N->getValueType(0)) 505 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), 506 Res); 507 return Res; 508 } 509 510 511 /// ScalarizeVecOp_VSELECT - If the input condition is a vector that needs to be 512 /// scalarized, it must be <1 x i1>, so just convert to a normal ISD::SELECT 513 /// (still with vector output type since that was acceptable if we got here). 514 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) { 515 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0)); 516 EVT VT = N->getValueType(0); 517 518 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1), 519 N->getOperand(2)); 520 } 521 522 /// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 523 /// scalarized, it must be <1 x ty>. Just store the element. 524 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 525 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 526 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 527 SDLoc dl(N); 528 529 if (N->isTruncatingStore()) 530 return DAG.getTruncStore(N->getChain(), dl, 531 GetScalarizedVector(N->getOperand(1)), 532 N->getBasePtr(), N->getPointerInfo(), 533 N->getMemoryVT().getVectorElementType(), 534 N->isVolatile(), N->isNonTemporal(), 535 N->getAlignment(), N->getAAInfo()); 536 537 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 538 N->getBasePtr(), N->getPointerInfo(), 539 N->isVolatile(), N->isNonTemporal(), 540 N->getOriginalAlignment(), N->getAAInfo()); 541 } 542 543 /// ScalarizeVecOp_FP_ROUND - If the value to round is a vector that needs 544 /// to be scalarized, it must be <1 x ty>. Convert the element instead. 545 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) { 546 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 547 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N), 548 N->getValueType(0).getVectorElementType(), Elt, 549 N->getOperand(1)); 550 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 551 } 552 553 //===----------------------------------------------------------------------===// 554 // Result Vector Splitting 555 //===----------------------------------------------------------------------===// 556 557 /// SplitVectorResult - This method is called when the specified result of the 558 /// specified node is found to need vector splitting. At this point, the node 559 /// may also have invalid operands or may have other results that need 560 /// legalization, we just know that (at least) one result needs vector 561 /// splitting. 562 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 563 DEBUG(dbgs() << "Split node result: "; 564 N->dump(&DAG); 565 dbgs() << "\n"); 566 SDValue Lo, Hi; 567 568 // See if the target wants to custom expand this node. 569 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 570 return; 571 572 switch (N->getOpcode()) { 573 default: 574 #ifndef NDEBUG 575 dbgs() << "SplitVectorResult #" << ResNo << ": "; 576 N->dump(&DAG); 577 dbgs() << "\n"; 578 #endif 579 report_fatal_error("Do not know how to split the result of this " 580 "operator!\n"); 581 582 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 583 case ISD::VSELECT: 584 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 585 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 586 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 587 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break; 588 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 589 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 590 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 591 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break; 592 case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 593 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 594 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 595 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 596 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 597 case ISD::LOAD: 598 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 599 break; 600 case ISD::MLOAD: 601 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi); 602 break; 603 case ISD::SETCC: 604 SplitVecRes_SETCC(N, Lo, Hi); 605 break; 606 case ISD::VECTOR_SHUFFLE: 607 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 608 break; 609 610 case ISD::BSWAP: 611 case ISD::CONVERT_RNDSAT: 612 case ISD::CTLZ: 613 case ISD::CTTZ: 614 case ISD::CTLZ_ZERO_UNDEF: 615 case ISD::CTTZ_ZERO_UNDEF: 616 case ISD::CTPOP: 617 case ISD::FABS: 618 case ISD::FCEIL: 619 case ISD::FCOS: 620 case ISD::FEXP: 621 case ISD::FEXP2: 622 case ISD::FFLOOR: 623 case ISD::FLOG: 624 case ISD::FLOG10: 625 case ISD::FLOG2: 626 case ISD::FNEARBYINT: 627 case ISD::FNEG: 628 case ISD::FP_EXTEND: 629 case ISD::FP_ROUND: 630 case ISD::FP_TO_SINT: 631 case ISD::FP_TO_UINT: 632 case ISD::FRINT: 633 case ISD::FROUND: 634 case ISD::FSIN: 635 case ISD::FSQRT: 636 case ISD::FTRUNC: 637 case ISD::SINT_TO_FP: 638 case ISD::TRUNCATE: 639 case ISD::UINT_TO_FP: 640 SplitVecRes_UnaryOp(N, Lo, Hi); 641 break; 642 643 case ISD::ANY_EXTEND: 644 case ISD::SIGN_EXTEND: 645 case ISD::ZERO_EXTEND: 646 SplitVecRes_ExtendOp(N, Lo, Hi); 647 break; 648 649 case ISD::ADD: 650 case ISD::SUB: 651 case ISD::MUL: 652 case ISD::FADD: 653 case ISD::FCOPYSIGN: 654 case ISD::FSUB: 655 case ISD::FMUL: 656 case ISD::FMINNUM: 657 case ISD::FMAXNUM: 658 case ISD::SDIV: 659 case ISD::UDIV: 660 case ISD::FDIV: 661 case ISD::FPOW: 662 case ISD::AND: 663 case ISD::OR: 664 case ISD::XOR: 665 case ISD::SHL: 666 case ISD::SRA: 667 case ISD::SRL: 668 case ISD::UREM: 669 case ISD::SREM: 670 case ISD::FREM: 671 SplitVecRes_BinOp(N, Lo, Hi); 672 break; 673 case ISD::FMA: 674 SplitVecRes_TernaryOp(N, Lo, Hi); 675 break; 676 } 677 678 // If Lo/Hi is null, the sub-method took care of registering results etc. 679 if (Lo.getNode()) 680 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 681 } 682 683 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 684 SDValue &Hi) { 685 SDValue LHSLo, LHSHi; 686 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 687 SDValue RHSLo, RHSHi; 688 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 689 SDLoc dl(N); 690 691 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 692 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 693 } 694 695 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, 696 SDValue &Hi) { 697 SDValue Op0Lo, Op0Hi; 698 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 699 SDValue Op1Lo, Op1Hi; 700 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 701 SDValue Op2Lo, Op2Hi; 702 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi); 703 SDLoc dl(N); 704 705 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), 706 Op0Lo, Op1Lo, Op2Lo); 707 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), 708 Op0Hi, Op1Hi, Op2Hi); 709 } 710 711 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, 712 SDValue &Hi) { 713 // We know the result is a vector. The input may be either a vector or a 714 // scalar value. 715 EVT LoVT, HiVT; 716 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 717 SDLoc dl(N); 718 719 SDValue InOp = N->getOperand(0); 720 EVT InVT = InOp.getValueType(); 721 722 // Handle some special cases efficiently. 723 switch (getTypeAction(InVT)) { 724 case TargetLowering::TypeLegal: 725 case TargetLowering::TypePromoteInteger: 726 case TargetLowering::TypePromoteFloat: 727 case TargetLowering::TypeSoftenFloat: 728 case TargetLowering::TypeScalarizeVector: 729 case TargetLowering::TypeWidenVector: 730 break; 731 case TargetLowering::TypeExpandInteger: 732 case TargetLowering::TypeExpandFloat: 733 // A scalar to vector conversion, where the scalar needs expansion. 734 // If the vector is being split in two then we can just convert the 735 // expanded pieces. 736 if (LoVT == HiVT) { 737 GetExpandedOp(InOp, Lo, Hi); 738 if (TLI.isBigEndian()) 739 std::swap(Lo, Hi); 740 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 741 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 742 return; 743 } 744 break; 745 case TargetLowering::TypeSplitVector: 746 // If the input is a vector that needs to be split, convert each split 747 // piece of the input now. 748 GetSplitVector(InOp, Lo, Hi); 749 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 750 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 751 return; 752 } 753 754 // In the general case, convert the input to an integer and split it by hand. 755 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 756 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 757 if (TLI.isBigEndian()) 758 std::swap(LoIntVT, HiIntVT); 759 760 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 761 762 if (TLI.isBigEndian()) 763 std::swap(Lo, Hi); 764 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 765 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 766 } 767 768 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 769 SDValue &Hi) { 770 EVT LoVT, HiVT; 771 SDLoc dl(N); 772 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 773 unsigned LoNumElts = LoVT.getVectorNumElements(); 774 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 775 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, LoOps); 776 777 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 778 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, HiOps); 779 } 780 781 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 782 SDValue &Hi) { 783 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 784 SDLoc dl(N); 785 unsigned NumSubvectors = N->getNumOperands() / 2; 786 if (NumSubvectors == 1) { 787 Lo = N->getOperand(0); 788 Hi = N->getOperand(1); 789 return; 790 } 791 792 EVT LoVT, HiVT; 793 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 794 795 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 796 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps); 797 798 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 799 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps); 800 } 801 802 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 803 SDValue &Hi) { 804 SDValue Vec = N->getOperand(0); 805 SDValue Idx = N->getOperand(1); 806 SDLoc dl(N); 807 808 EVT LoVT, HiVT; 809 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 810 811 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 812 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 813 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, 814 DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), 815 TLI.getVectorIdxTy())); 816 } 817 818 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, 819 SDValue &Hi) { 820 SDValue Vec = N->getOperand(0); 821 SDValue SubVec = N->getOperand(1); 822 SDValue Idx = N->getOperand(2); 823 SDLoc dl(N); 824 GetSplitVector(Vec, Lo, Hi); 825 826 // Spill the vector to the stack. 827 EVT VecVT = Vec.getValueType(); 828 EVT SubVecVT = VecVT.getVectorElementType(); 829 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 830 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 831 MachinePointerInfo(), false, false, 0); 832 833 // Store the new subvector into the specified index. 834 SDValue SubVecPtr = GetVectorElementPointer(StackPtr, SubVecVT, Idx); 835 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 836 unsigned Alignment = TLI.getDataLayout()->getPrefTypeAlignment(VecType); 837 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo(), 838 false, false, 0); 839 840 // Load the Lo part from the stack slot. 841 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 842 false, false, false, 0); 843 844 // Increment the pointer to the other part. 845 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 846 StackPtr = 847 DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 848 DAG.getConstant(IncrementSize, StackPtr.getValueType())); 849 850 // Load the Hi part from the stack slot. 851 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 852 false, false, false, MinAlign(Alignment, IncrementSize)); 853 } 854 855 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 856 SDValue &Hi) { 857 SDLoc dl(N); 858 GetSplitVector(N->getOperand(0), Lo, Hi); 859 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 860 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 861 } 862 863 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 864 SDValue &Hi) { 865 SDValue LHSLo, LHSHi; 866 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 867 SDLoc dl(N); 868 869 EVT LoVT, HiVT; 870 std::tie(LoVT, HiVT) = 871 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT()); 872 873 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 874 DAG.getValueType(LoVT)); 875 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 876 DAG.getValueType(HiVT)); 877 } 878 879 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 880 SDValue &Hi) { 881 SDValue Vec = N->getOperand(0); 882 SDValue Elt = N->getOperand(1); 883 SDValue Idx = N->getOperand(2); 884 SDLoc dl(N); 885 GetSplitVector(Vec, Lo, Hi); 886 887 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 888 unsigned IdxVal = CIdx->getZExtValue(); 889 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 890 if (IdxVal < LoNumElts) 891 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 892 Lo.getValueType(), Lo, Elt, Idx); 893 else 894 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 895 DAG.getConstant(IdxVal - LoNumElts, 896 TLI.getVectorIdxTy())); 897 return; 898 } 899 900 // See if the target wants to custom expand this node. 901 if (CustomLowerNode(N, N->getValueType(0), true)) 902 return; 903 904 // Spill the vector to the stack. 905 EVT VecVT = Vec.getValueType(); 906 EVT EltVT = VecVT.getVectorElementType(); 907 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 908 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 909 MachinePointerInfo(), false, false, 0); 910 911 // Store the new element. This may be larger than the vector element type, 912 // so use a truncating store. 913 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 914 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 915 unsigned Alignment = 916 TLI.getDataLayout()->getPrefTypeAlignment(VecType); 917 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT, 918 false, false, 0); 919 920 // Load the Lo part from the stack slot. 921 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 922 false, false, false, 0); 923 924 // Increment the pointer to the other part. 925 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 926 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 927 DAG.getConstant(IncrementSize, StackPtr.getValueType())); 928 929 // Load the Hi part from the stack slot. 930 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 931 false, false, false, MinAlign(Alignment, IncrementSize)); 932 } 933 934 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 935 SDValue &Hi) { 936 EVT LoVT, HiVT; 937 SDLoc dl(N); 938 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 939 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 940 Hi = DAG.getUNDEF(HiVT); 941 } 942 943 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 944 SDValue &Hi) { 945 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 946 EVT LoVT, HiVT; 947 SDLoc dl(LD); 948 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0)); 949 950 ISD::LoadExtType ExtType = LD->getExtensionType(); 951 SDValue Ch = LD->getChain(); 952 SDValue Ptr = LD->getBasePtr(); 953 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 954 EVT MemoryVT = LD->getMemoryVT(); 955 unsigned Alignment = LD->getOriginalAlignment(); 956 bool isVolatile = LD->isVolatile(); 957 bool isNonTemporal = LD->isNonTemporal(); 958 bool isInvariant = LD->isInvariant(); 959 AAMDNodes AAInfo = LD->getAAInfo(); 960 961 EVT LoMemVT, HiMemVT; 962 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 963 964 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset, 965 LD->getPointerInfo(), LoMemVT, isVolatile, isNonTemporal, 966 isInvariant, Alignment, AAInfo); 967 968 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 969 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 970 DAG.getConstant(IncrementSize, Ptr.getValueType())); 971 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, 972 LD->getPointerInfo().getWithOffset(IncrementSize), 973 HiMemVT, isVolatile, isNonTemporal, isInvariant, Alignment, 974 AAInfo); 975 976 // Build a factor node to remember that this load is independent of the 977 // other one. 978 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 979 Hi.getValue(1)); 980 981 // Legalized the chain result - switch anything that used the old chain to 982 // use the new one. 983 ReplaceValueWith(SDValue(LD, 1), Ch); 984 } 985 986 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, 987 SDValue &Lo, SDValue &Hi) { 988 EVT LoVT, HiVT; 989 SDLoc dl(MLD); 990 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0)); 991 992 SDValue Ch = MLD->getChain(); 993 SDValue Ptr = MLD->getBasePtr(); 994 SDValue Mask = MLD->getMask(); 995 unsigned Alignment = MLD->getOriginalAlignment(); 996 ISD::LoadExtType ExtType = MLD->getExtensionType(); 997 998 // if Alignment is equal to the vector size, 999 // take the half of it for the second part 1000 unsigned SecondHalfAlignment = 1001 (Alignment == MLD->getValueType(0).getSizeInBits()/8) ? 1002 Alignment/2 : Alignment; 1003 1004 SDValue MaskLo, MaskHi; 1005 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 1006 1007 EVT MemoryVT = MLD->getMemoryVT(); 1008 EVT LoMemVT, HiMemVT; 1009 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1010 1011 SDValue Src0 = MLD->getSrc0(); 1012 SDValue Src0Lo, Src0Hi; 1013 std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl); 1014 1015 MachineMemOperand *MMO = DAG.getMachineFunction(). 1016 getMachineMemOperand(MLD->getPointerInfo(), 1017 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(), 1018 Alignment, MLD->getAAInfo(), MLD->getRanges()); 1019 1020 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, Src0Lo, LoMemVT, MMO, 1021 ExtType); 1022 1023 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1024 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1025 DAG.getConstant(IncrementSize, Ptr.getValueType())); 1026 1027 MMO = DAG.getMachineFunction(). 1028 getMachineMemOperand(MLD->getPointerInfo(), 1029 MachineMemOperand::MOLoad, HiMemVT.getStoreSize(), 1030 SecondHalfAlignment, MLD->getAAInfo(), MLD->getRanges()); 1031 1032 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, Src0Hi, HiMemVT, MMO, 1033 ExtType); 1034 1035 1036 // Build a factor node to remember that this load is independent of the 1037 // other one. 1038 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1039 Hi.getValue(1)); 1040 1041 // Legalized the chain result - switch anything that used the old chain to 1042 // use the new one. 1043 ReplaceValueWith(SDValue(MLD, 1), Ch); 1044 1045 } 1046 1047 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 1048 assert(N->getValueType(0).isVector() && 1049 N->getOperand(0).getValueType().isVector() && 1050 "Operand types must be vectors"); 1051 1052 EVT LoVT, HiVT; 1053 SDLoc DL(N); 1054 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1055 1056 // Split the input. 1057 SDValue LL, LH, RL, RH; 1058 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0); 1059 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1); 1060 1061 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 1062 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 1063 } 1064 1065 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 1066 SDValue &Hi) { 1067 // Get the dest types - they may not match the input types, e.g. int_to_fp. 1068 EVT LoVT, HiVT; 1069 SDLoc dl(N); 1070 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1071 1072 // If the input also splits, handle it directly for a compile time speedup. 1073 // Otherwise split it by hand. 1074 EVT InVT = N->getOperand(0).getValueType(); 1075 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) 1076 GetSplitVector(N->getOperand(0), Lo, Hi); 1077 else 1078 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); 1079 1080 if (N->getOpcode() == ISD::FP_ROUND) { 1081 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1)); 1082 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1)); 1083 } else if (N->getOpcode() == ISD::CONVERT_RNDSAT) { 1084 SDValue DTyOpLo = DAG.getValueType(LoVT); 1085 SDValue DTyOpHi = DAG.getValueType(HiVT); 1086 SDValue STyOpLo = DAG.getValueType(Lo.getValueType()); 1087 SDValue STyOpHi = DAG.getValueType(Hi.getValueType()); 1088 SDValue RndOp = N->getOperand(3); 1089 SDValue SatOp = N->getOperand(4); 1090 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1091 Lo = DAG.getConvertRndSat(LoVT, dl, Lo, DTyOpLo, STyOpLo, RndOp, SatOp, 1092 CvtCode); 1093 Hi = DAG.getConvertRndSat(HiVT, dl, Hi, DTyOpHi, STyOpHi, RndOp, SatOp, 1094 CvtCode); 1095 } else { 1096 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 1097 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 1098 } 1099 } 1100 1101 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, 1102 SDValue &Hi) { 1103 SDLoc dl(N); 1104 EVT SrcVT = N->getOperand(0).getValueType(); 1105 EVT DestVT = N->getValueType(0); 1106 EVT LoVT, HiVT; 1107 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT); 1108 1109 // We can do better than a generic split operation if the extend is doing 1110 // more than just doubling the width of the elements and the following are 1111 // true: 1112 // - The number of vector elements is even, 1113 // - the source type is legal, 1114 // - the type of a split source is illegal, 1115 // - the type of an extended (by doubling element size) source is legal, and 1116 // - the type of that extended source when split is legal. 1117 // 1118 // This won't necessarily completely legalize the operation, but it will 1119 // more effectively move in the right direction and prevent falling down 1120 // to scalarization in many cases due to the input vector being split too 1121 // far. 1122 unsigned NumElements = SrcVT.getVectorNumElements(); 1123 if ((NumElements & 1) == 0 && 1124 SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) { 1125 LLVMContext &Ctx = *DAG.getContext(); 1126 EVT NewSrcVT = EVT::getVectorVT( 1127 Ctx, EVT::getIntegerVT( 1128 Ctx, SrcVT.getVectorElementType().getSizeInBits() * 2), 1129 NumElements); 1130 EVT SplitSrcVT = 1131 EVT::getVectorVT(Ctx, SrcVT.getVectorElementType(), NumElements / 2); 1132 EVT SplitLoVT, SplitHiVT; 1133 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT); 1134 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) && 1135 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) { 1136 DEBUG(dbgs() << "Split vector extend via incremental extend:"; 1137 N->dump(&DAG); dbgs() << "\n"); 1138 // Extend the source vector by one step. 1139 SDValue NewSrc = 1140 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0)); 1141 // Get the low and high halves of the new, extended one step, vector. 1142 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl); 1143 // Extend those vector halves the rest of the way. 1144 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 1145 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 1146 return; 1147 } 1148 } 1149 // Fall back to the generic unary operator splitting otherwise. 1150 SplitVecRes_UnaryOp(N, Lo, Hi); 1151 } 1152 1153 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 1154 SDValue &Lo, SDValue &Hi) { 1155 // The low and high parts of the original input give four input vectors. 1156 SDValue Inputs[4]; 1157 SDLoc dl(N); 1158 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 1159 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 1160 EVT NewVT = Inputs[0].getValueType(); 1161 unsigned NewElts = NewVT.getVectorNumElements(); 1162 1163 // If Lo or Hi uses elements from at most two of the four input vectors, then 1164 // express it as a vector shuffle of those two inputs. Otherwise extract the 1165 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 1166 SmallVector<int, 16> Ops; 1167 for (unsigned High = 0; High < 2; ++High) { 1168 SDValue &Output = High ? Hi : Lo; 1169 1170 // Build a shuffle mask for the output, discovering on the fly which 1171 // input vectors to use as shuffle operands (recorded in InputUsed). 1172 // If building a suitable shuffle vector proves too hard, then bail 1173 // out with useBuildVector set. 1174 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 1175 unsigned FirstMaskIdx = High * NewElts; 1176 bool useBuildVector = false; 1177 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 1178 // The mask element. This indexes into the input. 1179 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 1180 1181 // The input vector this mask element indexes into. 1182 unsigned Input = (unsigned)Idx / NewElts; 1183 1184 if (Input >= array_lengthof(Inputs)) { 1185 // The mask element does not index into any input vector. 1186 Ops.push_back(-1); 1187 continue; 1188 } 1189 1190 // Turn the index into an offset from the start of the input vector. 1191 Idx -= Input * NewElts; 1192 1193 // Find or create a shuffle vector operand to hold this input. 1194 unsigned OpNo; 1195 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 1196 if (InputUsed[OpNo] == Input) { 1197 // This input vector is already an operand. 1198 break; 1199 } else if (InputUsed[OpNo] == -1U) { 1200 // Create a new operand for this input vector. 1201 InputUsed[OpNo] = Input; 1202 break; 1203 } 1204 } 1205 1206 if (OpNo >= array_lengthof(InputUsed)) { 1207 // More than two input vectors used! Give up on trying to create a 1208 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 1209 useBuildVector = true; 1210 break; 1211 } 1212 1213 // Add the mask index for the new shuffle vector. 1214 Ops.push_back(Idx + OpNo * NewElts); 1215 } 1216 1217 if (useBuildVector) { 1218 EVT EltVT = NewVT.getVectorElementType(); 1219 SmallVector<SDValue, 16> SVOps; 1220 1221 // Extract the input elements by hand. 1222 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 1223 // The mask element. This indexes into the input. 1224 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 1225 1226 // The input vector this mask element indexes into. 1227 unsigned Input = (unsigned)Idx / NewElts; 1228 1229 if (Input >= array_lengthof(Inputs)) { 1230 // The mask element is "undef" or indexes off the end of the input. 1231 SVOps.push_back(DAG.getUNDEF(EltVT)); 1232 continue; 1233 } 1234 1235 // Turn the index into an offset from the start of the input vector. 1236 Idx -= Input * NewElts; 1237 1238 // Extract the vector element by hand. 1239 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 1240 Inputs[Input], DAG.getConstant(Idx, 1241 TLI.getVectorIdxTy()))); 1242 } 1243 1244 // Construct the Lo/Hi output using a BUILD_VECTOR. 1245 Output = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, SVOps); 1246 } else if (InputUsed[0] == -1U) { 1247 // No input vectors were used! The result is undefined. 1248 Output = DAG.getUNDEF(NewVT); 1249 } else { 1250 SDValue Op0 = Inputs[InputUsed[0]]; 1251 // If only one input was used, use an undefined vector for the other. 1252 SDValue Op1 = InputUsed[1] == -1U ? 1253 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 1254 // At least one input vector was used. Create a new shuffle vector. 1255 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 1256 } 1257 1258 Ops.clear(); 1259 } 1260 } 1261 1262 1263 //===----------------------------------------------------------------------===// 1264 // Operand Vector Splitting 1265 //===----------------------------------------------------------------------===// 1266 1267 /// SplitVectorOperand - This method is called when the specified operand of the 1268 /// specified node is found to need vector splitting. At this point, all of the 1269 /// result types of the node are known to be legal, but other operands of the 1270 /// node may need legalization as well as the specified one. 1271 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 1272 DEBUG(dbgs() << "Split node operand: "; 1273 N->dump(&DAG); 1274 dbgs() << "\n"); 1275 SDValue Res = SDValue(); 1276 1277 // See if the target wants to custom split this node. 1278 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 1279 return false; 1280 1281 if (!Res.getNode()) { 1282 switch (N->getOpcode()) { 1283 default: 1284 #ifndef NDEBUG 1285 dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 1286 N->dump(&DAG); 1287 dbgs() << "\n"; 1288 #endif 1289 report_fatal_error("Do not know how to split this operator's " 1290 "operand!\n"); 1291 1292 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; 1293 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; 1294 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 1295 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 1296 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; 1297 case ISD::TRUNCATE: 1298 Res = SplitVecOp_TruncateHelper(N); 1299 break; 1300 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; 1301 case ISD::STORE: 1302 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 1303 break; 1304 case ISD::MSTORE: 1305 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo); 1306 break; 1307 case ISD::VSELECT: 1308 Res = SplitVecOp_VSELECT(N, OpNo); 1309 break; 1310 case ISD::FP_TO_SINT: 1311 case ISD::FP_TO_UINT: 1312 if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0))) 1313 Res = SplitVecOp_TruncateHelper(N); 1314 else 1315 Res = SplitVecOp_UnaryOp(N); 1316 break; 1317 case ISD::SINT_TO_FP: 1318 case ISD::UINT_TO_FP: 1319 if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0))) 1320 Res = SplitVecOp_TruncateHelper(N); 1321 else 1322 Res = SplitVecOp_UnaryOp(N); 1323 break; 1324 case ISD::CTTZ: 1325 case ISD::CTLZ: 1326 case ISD::CTPOP: 1327 case ISD::FP_EXTEND: 1328 case ISD::SIGN_EXTEND: 1329 case ISD::ZERO_EXTEND: 1330 case ISD::ANY_EXTEND: 1331 case ISD::FTRUNC: 1332 Res = SplitVecOp_UnaryOp(N); 1333 break; 1334 } 1335 } 1336 1337 // If the result is null, the sub-method took care of registering results etc. 1338 if (!Res.getNode()) return false; 1339 1340 // If the result is N, the sub-method updated N in place. Tell the legalizer 1341 // core about this. 1342 if (Res.getNode() == N) 1343 return true; 1344 1345 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1346 "Invalid operand expansion"); 1347 1348 ReplaceValueWith(SDValue(N, 0), Res); 1349 return false; 1350 } 1351 1352 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { 1353 // The only possibility for an illegal operand is the mask, since result type 1354 // legalization would have handled this node already otherwise. 1355 assert(OpNo == 0 && "Illegal operand must be mask"); 1356 1357 SDValue Mask = N->getOperand(0); 1358 SDValue Src0 = N->getOperand(1); 1359 SDValue Src1 = N->getOperand(2); 1360 EVT Src0VT = Src0.getValueType(); 1361 SDLoc DL(N); 1362 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?"); 1363 1364 SDValue Lo, Hi; 1365 GetSplitVector(N->getOperand(0), Lo, Hi); 1366 assert(Lo.getValueType() == Hi.getValueType() && 1367 "Lo and Hi have differing types"); 1368 1369 EVT LoOpVT, HiOpVT; 1370 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT); 1371 assert(LoOpVT == HiOpVT && "Asymmetric vector split?"); 1372 1373 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask; 1374 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL); 1375 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL); 1376 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL); 1377 1378 SDValue LoSelect = 1379 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1); 1380 SDValue HiSelect = 1381 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1); 1382 1383 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect); 1384 } 1385 1386 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 1387 // The result has a legal vector type, but the input needs splitting. 1388 EVT ResVT = N->getValueType(0); 1389 SDValue Lo, Hi; 1390 SDLoc dl(N); 1391 GetSplitVector(N->getOperand(0), Lo, Hi); 1392 EVT InVT = Lo.getValueType(); 1393 1394 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1395 InVT.getVectorNumElements()); 1396 1397 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 1398 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 1399 1400 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 1401 } 1402 1403 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { 1404 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will 1405 // end up being split all the way down to individual components. Convert the 1406 // split pieces into integers and reassemble. 1407 SDValue Lo, Hi; 1408 GetSplitVector(N->getOperand(0), Lo, Hi); 1409 Lo = BitConvertToInteger(Lo); 1410 Hi = BitConvertToInteger(Hi); 1411 1412 if (TLI.isBigEndian()) 1413 std::swap(Lo, Hi); 1414 1415 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), 1416 JoinIntegers(Lo, Hi)); 1417 } 1418 1419 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1420 // We know that the extracted result type is legal. 1421 EVT SubVT = N->getValueType(0); 1422 SDValue Idx = N->getOperand(1); 1423 SDLoc dl(N); 1424 SDValue Lo, Hi; 1425 GetSplitVector(N->getOperand(0), Lo, Hi); 1426 1427 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1428 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1429 1430 if (IdxVal < LoElts) { 1431 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1432 "Extracted subvector crosses vector split!"); 1433 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1434 } else { 1435 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1436 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1437 } 1438 } 1439 1440 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1441 SDValue Vec = N->getOperand(0); 1442 SDValue Idx = N->getOperand(1); 1443 EVT VecVT = Vec.getValueType(); 1444 1445 if (isa<ConstantSDNode>(Idx)) { 1446 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1447 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1448 1449 SDValue Lo, Hi; 1450 GetSplitVector(Vec, Lo, Hi); 1451 1452 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1453 1454 if (IdxVal < LoElts) 1455 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0); 1456 return SDValue(DAG.UpdateNodeOperands(N, Hi, 1457 DAG.getConstant(IdxVal - LoElts, 1458 Idx.getValueType())), 0); 1459 } 1460 1461 // See if the target wants to custom expand this node. 1462 if (CustomLowerNode(N, N->getValueType(0), true)) 1463 return SDValue(); 1464 1465 // Store the vector to the stack. 1466 EVT EltVT = VecVT.getVectorElementType(); 1467 SDLoc dl(N); 1468 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1469 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1470 MachinePointerInfo(), false, false, 0); 1471 1472 // Load back the required element. 1473 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1474 return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 1475 MachinePointerInfo(), EltVT, false, false, false, 0); 1476 } 1477 1478 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N, 1479 unsigned OpNo) { 1480 SDValue Ch = N->getChain(); 1481 SDValue Ptr = N->getBasePtr(); 1482 SDValue Mask = N->getMask(); 1483 SDValue Data = N->getValue(); 1484 EVT MemoryVT = N->getMemoryVT(); 1485 unsigned Alignment = N->getOriginalAlignment(); 1486 SDLoc DL(N); 1487 1488 EVT LoMemVT, HiMemVT; 1489 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1490 1491 SDValue DataLo, DataHi; 1492 GetSplitVector(Data, DataLo, DataHi); 1493 SDValue MaskLo, MaskHi; 1494 GetSplitVector(Mask, MaskLo, MaskHi); 1495 1496 // if Alignment is equal to the vector size, 1497 // take the half of it for the second part 1498 unsigned SecondHalfAlignment = 1499 (Alignment == Data->getValueType(0).getSizeInBits()/8) ? 1500 Alignment/2 : Alignment; 1501 1502 SDValue Lo, Hi; 1503 MachineMemOperand *MMO = DAG.getMachineFunction(). 1504 getMachineMemOperand(N->getPointerInfo(), 1505 MachineMemOperand::MOStore, LoMemVT.getStoreSize(), 1506 Alignment, N->getAAInfo(), N->getRanges()); 1507 1508 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO, 1509 N->isTruncatingStore()); 1510 1511 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1512 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 1513 DAG.getConstant(IncrementSize, Ptr.getValueType())); 1514 1515 MMO = DAG.getMachineFunction(). 1516 getMachineMemOperand(N->getPointerInfo(), 1517 MachineMemOperand::MOStore, HiMemVT.getStoreSize(), 1518 SecondHalfAlignment, N->getAAInfo(), N->getRanges()); 1519 1520 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO, 1521 N->isTruncatingStore()); 1522 1523 1524 // Build a factor node to remember that this store is independent of the 1525 // other one. 1526 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 1527 1528 } 1529 1530 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1531 assert(N->isUnindexed() && "Indexed store of vector?"); 1532 assert(OpNo == 1 && "Can only split the stored value"); 1533 SDLoc DL(N); 1534 1535 bool isTruncating = N->isTruncatingStore(); 1536 SDValue Ch = N->getChain(); 1537 SDValue Ptr = N->getBasePtr(); 1538 EVT MemoryVT = N->getMemoryVT(); 1539 unsigned Alignment = N->getOriginalAlignment(); 1540 bool isVol = N->isVolatile(); 1541 bool isNT = N->isNonTemporal(); 1542 AAMDNodes AAInfo = N->getAAInfo(); 1543 SDValue Lo, Hi; 1544 GetSplitVector(N->getOperand(1), Lo, Hi); 1545 1546 EVT LoMemVT, HiMemVT; 1547 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1548 1549 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1550 1551 if (isTruncating) 1552 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1553 LoMemVT, isVol, isNT, Alignment, AAInfo); 1554 else 1555 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1556 isVol, isNT, Alignment, AAInfo); 1557 1558 // Increment the pointer to the other half. 1559 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 1560 DAG.getConstant(IncrementSize, Ptr.getValueType())); 1561 1562 if (isTruncating) 1563 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, 1564 N->getPointerInfo().getWithOffset(IncrementSize), 1565 HiMemVT, isVol, isNT, Alignment, AAInfo); 1566 else 1567 Hi = DAG.getStore(Ch, DL, Hi, Ptr, 1568 N->getPointerInfo().getWithOffset(IncrementSize), 1569 isVol, isNT, Alignment, AAInfo); 1570 1571 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 1572 } 1573 1574 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { 1575 SDLoc DL(N); 1576 1577 // The input operands all must have the same type, and we know the result 1578 // type is valid. Convert this to a buildvector which extracts all the 1579 // input elements. 1580 // TODO: If the input elements are power-two vectors, we could convert this to 1581 // a new CONCAT_VECTORS node with elements that are half-wide. 1582 SmallVector<SDValue, 32> Elts; 1583 EVT EltVT = N->getValueType(0).getVectorElementType(); 1584 for (unsigned op = 0, e = N->getNumOperands(); op != e; ++op) { 1585 SDValue Op = N->getOperand(op); 1586 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements(); 1587 i != e; ++i) { 1588 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, 1589 Op, DAG.getConstant(i, TLI.getVectorIdxTy()))); 1590 1591 } 1592 } 1593 1594 return DAG.getNode(ISD::BUILD_VECTOR, DL, N->getValueType(0), Elts); 1595 } 1596 1597 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) { 1598 // The result type is legal, but the input type is illegal. If splitting 1599 // ends up with the result type of each half still being legal, just 1600 // do that. If, however, that would result in an illegal result type, 1601 // we can try to get more clever with power-two vectors. Specifically, 1602 // split the input type, but also widen the result element size, then 1603 // concatenate the halves and truncate again. For example, consider a target 1604 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit 1605 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do: 1606 // %inlo = v4i32 extract_subvector %in, 0 1607 // %inhi = v4i32 extract_subvector %in, 4 1608 // %lo16 = v4i16 trunc v4i32 %inlo 1609 // %hi16 = v4i16 trunc v4i32 %inhi 1610 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16 1611 // %res = v8i8 trunc v8i16 %in16 1612 // 1613 // Without this transform, the original truncate would end up being 1614 // scalarized, which is pretty much always a last resort. 1615 SDValue InVec = N->getOperand(0); 1616 EVT InVT = InVec->getValueType(0); 1617 EVT OutVT = N->getValueType(0); 1618 unsigned NumElements = OutVT.getVectorNumElements(); 1619 bool IsFloat = OutVT.isFloatingPoint(); 1620 1621 // Widening should have already made sure this is a power-two vector 1622 // if we're trying to split it at all. assert() that's true, just in case. 1623 assert(!(NumElements & 1) && "Splitting vector, but not in half!"); 1624 1625 unsigned InElementSize = InVT.getVectorElementType().getSizeInBits(); 1626 unsigned OutElementSize = OutVT.getVectorElementType().getSizeInBits(); 1627 1628 // If the input elements are only 1/2 the width of the result elements, 1629 // just use the normal splitting. Our trick only work if there's room 1630 // to split more than once. 1631 if (InElementSize <= OutElementSize * 2) 1632 return SplitVecOp_UnaryOp(N); 1633 SDLoc DL(N); 1634 1635 // Extract the halves of the input via extract_subvector. 1636 SDValue InLoVec, InHiVec; 1637 std::tie(InLoVec, InHiVec) = DAG.SplitVector(InVec, DL); 1638 // Truncate them to 1/2 the element size. 1639 EVT HalfElementVT = IsFloat ? 1640 EVT::getFloatingPointVT(InElementSize/2) : 1641 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2); 1642 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, 1643 NumElements/2); 1644 SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec); 1645 SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec); 1646 // Concatenate them to get the full intermediate truncation result. 1647 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements); 1648 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo, 1649 HalfHi); 1650 // Now finish up by truncating all the way down to the original result 1651 // type. This should normally be something that ends up being legal directly, 1652 // but in theory if a target has very wide vectors and an annoyingly 1653 // restricted set of legal types, this split can chain to build things up. 1654 return IsFloat ? 1655 DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec, 1656 DAG.getTargetConstant(0, TLI.getPointerTy())) : 1657 DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec); 1658 } 1659 1660 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { 1661 assert(N->getValueType(0).isVector() && 1662 N->getOperand(0).getValueType().isVector() && 1663 "Operand types must be vectors"); 1664 // The result has a legal vector type, but the input needs splitting. 1665 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; 1666 SDLoc DL(N); 1667 GetSplitVector(N->getOperand(0), Lo0, Hi0); 1668 GetSplitVector(N->getOperand(1), Lo1, Hi1); 1669 unsigned PartElements = Lo0.getValueType().getVectorNumElements(); 1670 EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements); 1671 EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements); 1672 1673 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2)); 1674 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2)); 1675 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes); 1676 return PromoteTargetBoolean(Con, N->getValueType(0)); 1677 } 1678 1679 1680 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { 1681 // The result has a legal vector type, but the input needs splitting. 1682 EVT ResVT = N->getValueType(0); 1683 SDValue Lo, Hi; 1684 SDLoc DL(N); 1685 GetSplitVector(N->getOperand(0), Lo, Hi); 1686 EVT InVT = Lo.getValueType(); 1687 1688 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1689 InVT.getVectorNumElements()); 1690 1691 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); 1692 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); 1693 1694 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); 1695 } 1696 1697 1698 1699 //===----------------------------------------------------------------------===// 1700 // Result Vector Widening 1701 //===----------------------------------------------------------------------===// 1702 1703 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1704 DEBUG(dbgs() << "Widen node result " << ResNo << ": "; 1705 N->dump(&DAG); 1706 dbgs() << "\n"); 1707 1708 // See if the target wants to custom widen this node. 1709 if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 1710 return; 1711 1712 SDValue Res = SDValue(); 1713 switch (N->getOpcode()) { 1714 default: 1715 #ifndef NDEBUG 1716 dbgs() << "WidenVectorResult #" << ResNo << ": "; 1717 N->dump(&DAG); 1718 dbgs() << "\n"; 1719 #endif 1720 llvm_unreachable("Do not know how to widen the result of this operator!"); 1721 1722 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; 1723 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; 1724 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1725 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1726 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1727 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1728 case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break; 1729 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1730 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1731 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1732 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 1733 case ISD::VSELECT: 1734 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1735 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1736 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break; 1737 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1738 case ISD::VECTOR_SHUFFLE: 1739 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 1740 break; 1741 case ISD::MLOAD: 1742 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N)); 1743 break; 1744 1745 case ISD::ADD: 1746 case ISD::AND: 1747 case ISD::MUL: 1748 case ISD::MULHS: 1749 case ISD::MULHU: 1750 case ISD::OR: 1751 case ISD::SUB: 1752 case ISD::XOR: 1753 case ISD::FMINNUM: 1754 case ISD::FMAXNUM: 1755 Res = WidenVecRes_Binary(N); 1756 break; 1757 1758 case ISD::FADD: 1759 case ISD::FCOPYSIGN: 1760 case ISD::FMUL: 1761 case ISD::FPOW: 1762 case ISD::FSUB: 1763 case ISD::FDIV: 1764 case ISD::FREM: 1765 case ISD::SDIV: 1766 case ISD::UDIV: 1767 case ISD::SREM: 1768 case ISD::UREM: 1769 Res = WidenVecRes_BinaryCanTrap(N); 1770 break; 1771 1772 case ISD::FPOWI: 1773 Res = WidenVecRes_POWI(N); 1774 break; 1775 1776 case ISD::SHL: 1777 case ISD::SRA: 1778 case ISD::SRL: 1779 Res = WidenVecRes_Shift(N); 1780 break; 1781 1782 case ISD::ANY_EXTEND: 1783 case ISD::FP_EXTEND: 1784 case ISD::FP_ROUND: 1785 case ISD::FP_TO_SINT: 1786 case ISD::FP_TO_UINT: 1787 case ISD::SIGN_EXTEND: 1788 case ISD::SINT_TO_FP: 1789 case ISD::TRUNCATE: 1790 case ISD::UINT_TO_FP: 1791 case ISD::ZERO_EXTEND: 1792 Res = WidenVecRes_Convert(N); 1793 break; 1794 1795 case ISD::BSWAP: 1796 case ISD::CTLZ: 1797 case ISD::CTPOP: 1798 case ISD::CTTZ: 1799 case ISD::FABS: 1800 case ISD::FCEIL: 1801 case ISD::FCOS: 1802 case ISD::FEXP: 1803 case ISD::FEXP2: 1804 case ISD::FFLOOR: 1805 case ISD::FLOG: 1806 case ISD::FLOG10: 1807 case ISD::FLOG2: 1808 case ISD::FNEARBYINT: 1809 case ISD::FNEG: 1810 case ISD::FRINT: 1811 case ISD::FROUND: 1812 case ISD::FSIN: 1813 case ISD::FSQRT: 1814 case ISD::FTRUNC: 1815 Res = WidenVecRes_Unary(N); 1816 break; 1817 case ISD::FMA: 1818 Res = WidenVecRes_Ternary(N); 1819 break; 1820 } 1821 1822 // If Res is null, the sub-method took care of registering the result. 1823 if (Res.getNode()) 1824 SetWidenedVector(SDValue(N, ResNo), Res); 1825 } 1826 1827 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { 1828 // Ternary op widening. 1829 SDLoc dl(N); 1830 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1831 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1832 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1833 SDValue InOp3 = GetWidenedVector(N->getOperand(2)); 1834 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3); 1835 } 1836 1837 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1838 // Binary op widening. 1839 SDLoc dl(N); 1840 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1841 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1842 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1843 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2); 1844 } 1845 1846 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) { 1847 // Binary op widening for operations that can trap. 1848 unsigned Opcode = N->getOpcode(); 1849 SDLoc dl(N); 1850 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1851 EVT WidenEltVT = WidenVT.getVectorElementType(); 1852 EVT VT = WidenVT; 1853 unsigned NumElts = VT.getVectorNumElements(); 1854 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 1855 NumElts = NumElts / 2; 1856 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1857 } 1858 1859 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) { 1860 // Operation doesn't trap so just widen as normal. 1861 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1862 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1863 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2); 1864 } 1865 1866 // No legal vector version so unroll the vector operation and then widen. 1867 if (NumElts == 1) 1868 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 1869 1870 // Since the operation can trap, apply operation on the original vector. 1871 EVT MaxVT = VT; 1872 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1873 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1874 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 1875 1876 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 1877 unsigned ConcatEnd = 0; // Current ConcatOps index. 1878 int Idx = 0; // Current Idx into input vectors. 1879 1880 // NumElts := greatest legal vector size (at most WidenVT) 1881 // while (orig. vector has unhandled elements) { 1882 // take munches of size NumElts from the beginning and add to ConcatOps 1883 // NumElts := next smaller supported vector size or 1 1884 // } 1885 while (CurNumElts != 0) { 1886 while (CurNumElts >= NumElts) { 1887 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1, 1888 DAG.getConstant(Idx, TLI.getVectorIdxTy())); 1889 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2, 1890 DAG.getConstant(Idx, TLI.getVectorIdxTy())); 1891 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2); 1892 Idx += NumElts; 1893 CurNumElts -= NumElts; 1894 } 1895 do { 1896 NumElts = NumElts / 2; 1897 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1898 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 1899 1900 if (NumElts == 1) { 1901 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 1902 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1903 InOp1, DAG.getConstant(Idx, 1904 TLI.getVectorIdxTy())); 1905 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1906 InOp2, DAG.getConstant(Idx, 1907 TLI.getVectorIdxTy())); 1908 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT, 1909 EOp1, EOp2); 1910 } 1911 CurNumElts = 0; 1912 } 1913 } 1914 1915 // Check to see if we have a single operation with the widen type. 1916 if (ConcatEnd == 1) { 1917 VT = ConcatOps[0].getValueType(); 1918 if (VT == WidenVT) 1919 return ConcatOps[0]; 1920 } 1921 1922 // while (Some element of ConcatOps is not of type MaxVT) { 1923 // From the end of ConcatOps, collect elements of the same type and put 1924 // them into an op of the next larger supported type 1925 // } 1926 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { 1927 Idx = ConcatEnd - 1; 1928 VT = ConcatOps[Idx--].getValueType(); 1929 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) 1930 Idx--; 1931 1932 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1; 1933 EVT NextVT; 1934 do { 1935 NextSize *= 2; 1936 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize); 1937 } while (!TLI.isTypeLegal(NextVT)); 1938 1939 if (!VT.isVector()) { 1940 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT 1941 SDValue VecOp = DAG.getUNDEF(NextVT); 1942 unsigned NumToInsert = ConcatEnd - Idx - 1; 1943 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) { 1944 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, 1945 ConcatOps[OpIdx], DAG.getConstant(i, 1946 TLI.getVectorIdxTy())); 1947 } 1948 ConcatOps[Idx+1] = VecOp; 1949 ConcatEnd = Idx + 2; 1950 } else { 1951 // Vector type, create a CONCAT_VECTORS of type NextVT 1952 SDValue undefVec = DAG.getUNDEF(VT); 1953 unsigned OpsToConcat = NextSize/VT.getVectorNumElements(); 1954 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat); 1955 unsigned RealVals = ConcatEnd - Idx - 1; 1956 unsigned SubConcatEnd = 0; 1957 unsigned SubConcatIdx = Idx + 1; 1958 while (SubConcatEnd < RealVals) 1959 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx]; 1960 while (SubConcatEnd < OpsToConcat) 1961 SubConcatOps[SubConcatEnd++] = undefVec; 1962 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1963 NextVT, SubConcatOps); 1964 ConcatEnd = SubConcatIdx + 1; 1965 } 1966 } 1967 1968 // Check to see if we have a single operation with the widen type. 1969 if (ConcatEnd == 1) { 1970 VT = ConcatOps[0].getValueType(); 1971 if (VT == WidenVT) 1972 return ConcatOps[0]; 1973 } 1974 1975 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT 1976 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements(); 1977 if (NumOps != ConcatEnd ) { 1978 SDValue UndefVal = DAG.getUNDEF(MaxVT); 1979 for (unsigned j = ConcatEnd; j < NumOps; ++j) 1980 ConcatOps[j] = UndefVal; 1981 } 1982 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 1983 makeArrayRef(ConcatOps.data(), NumOps)); 1984 } 1985 1986 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1987 SDValue InOp = N->getOperand(0); 1988 SDLoc DL(N); 1989 1990 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1991 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1992 1993 EVT InVT = InOp.getValueType(); 1994 EVT InEltVT = InVT.getVectorElementType(); 1995 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1996 1997 unsigned Opcode = N->getOpcode(); 1998 unsigned InVTNumElts = InVT.getVectorNumElements(); 1999 2000 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 2001 InOp = GetWidenedVector(N->getOperand(0)); 2002 InVT = InOp.getValueType(); 2003 InVTNumElts = InVT.getVectorNumElements(); 2004 if (InVTNumElts == WidenNumElts) { 2005 if (N->getNumOperands() == 1) 2006 return DAG.getNode(Opcode, DL, WidenVT, InOp); 2007 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1)); 2008 } 2009 } 2010 2011 if (TLI.isTypeLegal(InWidenVT)) { 2012 // Because the result and the input are different vector types, widening 2013 // the result could create a legal type but widening the input might make 2014 // it an illegal type that might lead to repeatedly splitting the input 2015 // and then widening it. To avoid this, we widen the input only if 2016 // it results in a legal type. 2017 if (WidenNumElts % InVTNumElts == 0) { 2018 // Widen the input and call convert on the widened input vector. 2019 unsigned NumConcat = WidenNumElts/InVTNumElts; 2020 SmallVector<SDValue, 16> Ops(NumConcat); 2021 Ops[0] = InOp; 2022 SDValue UndefVal = DAG.getUNDEF(InVT); 2023 for (unsigned i = 1; i != NumConcat; ++i) 2024 Ops[i] = UndefVal; 2025 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops); 2026 if (N->getNumOperands() == 1) 2027 return DAG.getNode(Opcode, DL, WidenVT, InVec); 2028 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1)); 2029 } 2030 2031 if (InVTNumElts % WidenNumElts == 0) { 2032 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, 2033 InOp, DAG.getConstant(0, 2034 TLI.getVectorIdxTy())); 2035 // Extract the input and convert the shorten input vector. 2036 if (N->getNumOperands() == 1) 2037 return DAG.getNode(Opcode, DL, WidenVT, InVal); 2038 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1)); 2039 } 2040 } 2041 2042 // Otherwise unroll into some nasty scalar code and rebuild the vector. 2043 SmallVector<SDValue, 16> Ops(WidenNumElts); 2044 EVT EltVT = WidenVT.getVectorElementType(); 2045 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 2046 unsigned i; 2047 for (i=0; i < MinElts; ++i) { 2048 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 2049 DAG.getConstant(i, TLI.getVectorIdxTy())); 2050 if (N->getNumOperands() == 1) 2051 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val); 2052 else 2053 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1)); 2054 } 2055 2056 SDValue UndefVal = DAG.getUNDEF(EltVT); 2057 for (; i < WidenNumElts; ++i) 2058 Ops[i] = UndefVal; 2059 2060 return DAG.getNode(ISD::BUILD_VECTOR, DL, WidenVT, Ops); 2061 } 2062 2063 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) { 2064 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2065 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2066 SDValue ShOp = N->getOperand(1); 2067 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 2068 } 2069 2070 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 2071 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2072 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2073 SDValue ShOp = N->getOperand(1); 2074 2075 EVT ShVT = ShOp.getValueType(); 2076 if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) { 2077 ShOp = GetWidenedVector(ShOp); 2078 ShVT = ShOp.getValueType(); 2079 } 2080 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), 2081 ShVT.getVectorElementType(), 2082 WidenVT.getVectorNumElements()); 2083 if (ShVT != ShWidenVT) 2084 ShOp = ModifyToType(ShOp, ShWidenVT); 2085 2086 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 2087 } 2088 2089 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 2090 // Unary op widening. 2091 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2092 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2093 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp); 2094 } 2095 2096 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 2097 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2098 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 2099 cast<VTSDNode>(N->getOperand(1))->getVT() 2100 .getVectorElementType(), 2101 WidenVT.getVectorNumElements()); 2102 SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 2103 return DAG.getNode(N->getOpcode(), SDLoc(N), 2104 WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 2105 } 2106 2107 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { 2108 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); 2109 return GetWidenedVector(WidenVec); 2110 } 2111 2112 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { 2113 SDValue InOp = N->getOperand(0); 2114 EVT InVT = InOp.getValueType(); 2115 EVT VT = N->getValueType(0); 2116 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2117 SDLoc dl(N); 2118 2119 switch (getTypeAction(InVT)) { 2120 case TargetLowering::TypeLegal: 2121 break; 2122 case TargetLowering::TypePromoteInteger: 2123 // If the incoming type is a vector that is being promoted, then 2124 // we know that the elements are arranged differently and that we 2125 // must perform the conversion using a stack slot. 2126 if (InVT.isVector()) 2127 break; 2128 2129 // If the InOp is promoted to the same size, convert it. Otherwise, 2130 // fall out of the switch and widen the promoted input. 2131 InOp = GetPromotedInteger(InOp); 2132 InVT = InOp.getValueType(); 2133 if (WidenVT.bitsEq(InVT)) 2134 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 2135 break; 2136 case TargetLowering::TypeSoftenFloat: 2137 case TargetLowering::TypePromoteFloat: 2138 case TargetLowering::TypeExpandInteger: 2139 case TargetLowering::TypeExpandFloat: 2140 case TargetLowering::TypeScalarizeVector: 2141 case TargetLowering::TypeSplitVector: 2142 break; 2143 case TargetLowering::TypeWidenVector: 2144 // If the InOp is widened to the same size, convert it. Otherwise, fall 2145 // out of the switch and widen the widened input. 2146 InOp = GetWidenedVector(InOp); 2147 InVT = InOp.getValueType(); 2148 if (WidenVT.bitsEq(InVT)) 2149 // The input widens to the same size. Convert to the widen value. 2150 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 2151 break; 2152 } 2153 2154 unsigned WidenSize = WidenVT.getSizeInBits(); 2155 unsigned InSize = InVT.getSizeInBits(); 2156 // x86mmx is not an acceptable vector element type, so don't try. 2157 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) { 2158 // Determine new input vector type. The new input vector type will use 2159 // the same element type (if its a vector) or use the input type as a 2160 // vector. It is the same size as the type to widen to. 2161 EVT NewInVT; 2162 unsigned NewNumElts = WidenSize / InSize; 2163 if (InVT.isVector()) { 2164 EVT InEltVT = InVT.getVectorElementType(); 2165 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, 2166 WidenSize / InEltVT.getSizeInBits()); 2167 } else { 2168 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 2169 } 2170 2171 if (TLI.isTypeLegal(NewInVT)) { 2172 // Because the result and the input are different vector types, widening 2173 // the result could create a legal type but widening the input might make 2174 // it an illegal type that might lead to repeatedly splitting the input 2175 // and then widening it. To avoid this, we widen the input only if 2176 // it results in a legal type. 2177 SmallVector<SDValue, 16> Ops(NewNumElts); 2178 SDValue UndefVal = DAG.getUNDEF(InVT); 2179 Ops[0] = InOp; 2180 for (unsigned i = 1; i < NewNumElts; ++i) 2181 Ops[i] = UndefVal; 2182 2183 SDValue NewVec; 2184 if (InVT.isVector()) 2185 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops); 2186 else 2187 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, NewInVT, Ops); 2188 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec); 2189 } 2190 } 2191 2192 return CreateStackStoreLoad(InOp, WidenVT); 2193 } 2194 2195 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 2196 SDLoc dl(N); 2197 // Build a vector with undefined for the new nodes. 2198 EVT VT = N->getValueType(0); 2199 2200 // Integer BUILD_VECTOR operands may be larger than the node's vector element 2201 // type. The UNDEFs need to have the same type as the existing operands. 2202 EVT EltVT = N->getOperand(0).getValueType(); 2203 unsigned NumElts = VT.getVectorNumElements(); 2204 2205 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2206 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2207 2208 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 2209 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!"); 2210 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT)); 2211 2212 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, NewOps); 2213 } 2214 2215 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 2216 EVT InVT = N->getOperand(0).getValueType(); 2217 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2218 SDLoc dl(N); 2219 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2220 unsigned NumInElts = InVT.getVectorNumElements(); 2221 unsigned NumOperands = N->getNumOperands(); 2222 2223 bool InputWidened = false; // Indicates we need to widen the input. 2224 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) { 2225 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 2226 // Add undef vectors to widen to correct length. 2227 unsigned NumConcat = WidenVT.getVectorNumElements() / 2228 InVT.getVectorNumElements(); 2229 SDValue UndefVal = DAG.getUNDEF(InVT); 2230 SmallVector<SDValue, 16> Ops(NumConcat); 2231 for (unsigned i=0; i < NumOperands; ++i) 2232 Ops[i] = N->getOperand(i); 2233 for (unsigned i = NumOperands; i != NumConcat; ++i) 2234 Ops[i] = UndefVal; 2235 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops); 2236 } 2237 } else { 2238 InputWidened = true; 2239 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 2240 // The inputs and the result are widen to the same value. 2241 unsigned i; 2242 for (i=1; i < NumOperands; ++i) 2243 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 2244 break; 2245 2246 if (i == NumOperands) 2247 // Everything but the first operand is an UNDEF so just return the 2248 // widened first operand. 2249 return GetWidenedVector(N->getOperand(0)); 2250 2251 if (NumOperands == 2) { 2252 // Replace concat of two operands with a shuffle. 2253 SmallVector<int, 16> MaskOps(WidenNumElts, -1); 2254 for (unsigned i = 0; i < NumInElts; ++i) { 2255 MaskOps[i] = i; 2256 MaskOps[i + NumInElts] = i + WidenNumElts; 2257 } 2258 return DAG.getVectorShuffle(WidenVT, dl, 2259 GetWidenedVector(N->getOperand(0)), 2260 GetWidenedVector(N->getOperand(1)), 2261 &MaskOps[0]); 2262 } 2263 } 2264 } 2265 2266 // Fall back to use extracts and build vector. 2267 EVT EltVT = WidenVT.getVectorElementType(); 2268 SmallVector<SDValue, 16> Ops(WidenNumElts); 2269 unsigned Idx = 0; 2270 for (unsigned i=0; i < NumOperands; ++i) { 2271 SDValue InOp = N->getOperand(i); 2272 if (InputWidened) 2273 InOp = GetWidenedVector(InOp); 2274 for (unsigned j=0; j < NumInElts; ++j) 2275 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2276 DAG.getConstant(j, TLI.getVectorIdxTy())); 2277 } 2278 SDValue UndefVal = DAG.getUNDEF(EltVT); 2279 for (; Idx < WidenNumElts; ++Idx) 2280 Ops[Idx] = UndefVal; 2281 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2282 } 2283 2284 SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 2285 SDLoc dl(N); 2286 SDValue InOp = N->getOperand(0); 2287 SDValue RndOp = N->getOperand(3); 2288 SDValue SatOp = N->getOperand(4); 2289 2290 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2291 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2292 2293 EVT InVT = InOp.getValueType(); 2294 EVT InEltVT = InVT.getVectorElementType(); 2295 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 2296 2297 SDValue DTyOp = DAG.getValueType(WidenVT); 2298 SDValue STyOp = DAG.getValueType(InWidenVT); 2299 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 2300 2301 unsigned InVTNumElts = InVT.getVectorNumElements(); 2302 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 2303 InOp = GetWidenedVector(InOp); 2304 InVT = InOp.getValueType(); 2305 InVTNumElts = InVT.getVectorNumElements(); 2306 if (InVTNumElts == WidenNumElts) 2307 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2308 SatOp, CvtCode); 2309 } 2310 2311 if (TLI.isTypeLegal(InWidenVT)) { 2312 // Because the result and the input are different vector types, widening 2313 // the result could create a legal type but widening the input might make 2314 // it an illegal type that might lead to repeatedly splitting the input 2315 // and then widening it. To avoid this, we widen the input only if 2316 // it results in a legal type. 2317 if (WidenNumElts % InVTNumElts == 0) { 2318 // Widen the input and call convert on the widened input vector. 2319 unsigned NumConcat = WidenNumElts/InVTNumElts; 2320 SmallVector<SDValue, 16> Ops(NumConcat); 2321 Ops[0] = InOp; 2322 SDValue UndefVal = DAG.getUNDEF(InVT); 2323 for (unsigned i = 1; i != NumConcat; ++i) 2324 Ops[i] = UndefVal; 2325 2326 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, Ops); 2327 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2328 SatOp, CvtCode); 2329 } 2330 2331 if (InVTNumElts % WidenNumElts == 0) { 2332 // Extract the input and convert the shorten input vector. 2333 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 2334 DAG.getConstant(0, TLI.getVectorIdxTy())); 2335 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2336 SatOp, CvtCode); 2337 } 2338 } 2339 2340 // Otherwise unroll into some nasty scalar code and rebuild the vector. 2341 SmallVector<SDValue, 16> Ops(WidenNumElts); 2342 EVT EltVT = WidenVT.getVectorElementType(); 2343 DTyOp = DAG.getValueType(EltVT); 2344 STyOp = DAG.getValueType(InEltVT); 2345 2346 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 2347 unsigned i; 2348 for (i=0; i < MinElts; ++i) { 2349 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 2350 DAG.getConstant(i, TLI.getVectorIdxTy())); 2351 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 2352 SatOp, CvtCode); 2353 } 2354 2355 SDValue UndefVal = DAG.getUNDEF(EltVT); 2356 for (; i < WidenNumElts; ++i) 2357 Ops[i] = UndefVal; 2358 2359 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2360 } 2361 2362 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 2363 EVT VT = N->getValueType(0); 2364 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2365 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2366 SDValue InOp = N->getOperand(0); 2367 SDValue Idx = N->getOperand(1); 2368 SDLoc dl(N); 2369 2370 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2371 InOp = GetWidenedVector(InOp); 2372 2373 EVT InVT = InOp.getValueType(); 2374 2375 // Check if we can just return the input vector after widening. 2376 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2377 if (IdxVal == 0 && InVT == WidenVT) 2378 return InOp; 2379 2380 // Check if we can extract from the vector. 2381 unsigned InNumElts = InVT.getVectorNumElements(); 2382 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 2383 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 2384 2385 // We could try widening the input to the right length but for now, extract 2386 // the original elements, fill the rest with undefs and build a vector. 2387 SmallVector<SDValue, 16> Ops(WidenNumElts); 2388 EVT EltVT = VT.getVectorElementType(); 2389 unsigned NumElts = VT.getVectorNumElements(); 2390 unsigned i; 2391 for (i=0; i < NumElts; ++i) 2392 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2393 DAG.getConstant(IdxVal+i, TLI.getVectorIdxTy())); 2394 2395 SDValue UndefVal = DAG.getUNDEF(EltVT); 2396 for (; i < WidenNumElts; ++i) 2397 Ops[i] = UndefVal; 2398 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2399 } 2400 2401 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 2402 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2403 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), 2404 InOp.getValueType(), InOp, 2405 N->getOperand(1), N->getOperand(2)); 2406 } 2407 2408 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 2409 LoadSDNode *LD = cast<LoadSDNode>(N); 2410 ISD::LoadExtType ExtType = LD->getExtensionType(); 2411 2412 SDValue Result; 2413 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 2414 if (ExtType != ISD::NON_EXTLOAD) 2415 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); 2416 else 2417 Result = GenWidenVectorLoads(LdChain, LD); 2418 2419 // If we generate a single load, we can use that for the chain. Otherwise, 2420 // build a factor node to remember the multiple loads are independent and 2421 // chain to that. 2422 SDValue NewChain; 2423 if (LdChain.size() == 1) 2424 NewChain = LdChain[0]; 2425 else 2426 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain); 2427 2428 // Modified the chain - switch anything that used the old chain to use 2429 // the new one. 2430 ReplaceValueWith(SDValue(N, 1), NewChain); 2431 2432 return Result; 2433 } 2434 2435 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) { 2436 2437 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0)); 2438 SDValue Mask = N->getMask(); 2439 EVT MaskVT = Mask.getValueType(); 2440 SDValue Src0 = GetWidenedVector(N->getSrc0()); 2441 ISD::LoadExtType ExtType = N->getExtensionType(); 2442 SDLoc dl(N); 2443 2444 if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector) 2445 Mask = GetWidenedVector(Mask); 2446 else { 2447 EVT BoolVT = getSetCCResultType(WidenVT); 2448 2449 // We can't use ModifyToType() because we should fill the mask with 2450 // zeroes 2451 unsigned WidenNumElts = BoolVT.getVectorNumElements(); 2452 unsigned MaskNumElts = MaskVT.getVectorNumElements(); 2453 2454 unsigned NumConcat = WidenNumElts / MaskNumElts; 2455 SmallVector<SDValue, 16> Ops(NumConcat); 2456 SDValue ZeroVal = DAG.getConstant(0, MaskVT); 2457 Ops[0] = Mask; 2458 for (unsigned i = 1; i != NumConcat; ++i) 2459 Ops[i] = ZeroVal; 2460 2461 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops); 2462 } 2463 2464 SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(), 2465 Mask, Src0, N->getMemoryVT(), 2466 N->getMemOperand(), ExtType); 2467 // Legalized the chain result - switch anything that used the old chain to 2468 // use the new one. 2469 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2470 return Res; 2471 } 2472 2473 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 2474 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2475 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), 2476 WidenVT, N->getOperand(0)); 2477 } 2478 2479 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 2480 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2481 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2482 2483 SDValue Cond1 = N->getOperand(0); 2484 EVT CondVT = Cond1.getValueType(); 2485 if (CondVT.isVector()) { 2486 EVT CondEltVT = CondVT.getVectorElementType(); 2487 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), 2488 CondEltVT, WidenNumElts); 2489 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector) 2490 Cond1 = GetWidenedVector(Cond1); 2491 2492 // If we have to split the condition there is no point in widening the 2493 // select. This would result in an cycle of widening the select -> 2494 // widening the condition operand -> splitting the condition operand -> 2495 // splitting the select -> widening the select. Instead split this select 2496 // further and widen the resulting type. 2497 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) { 2498 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0); 2499 SDValue Res = ModifyToType(SplitSelect, WidenVT); 2500 return Res; 2501 } 2502 2503 if (Cond1.getValueType() != CondWidenVT) 2504 Cond1 = ModifyToType(Cond1, CondWidenVT); 2505 } 2506 2507 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 2508 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 2509 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 2510 return DAG.getNode(N->getOpcode(), SDLoc(N), 2511 WidenVT, Cond1, InOp1, InOp2); 2512 } 2513 2514 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 2515 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 2516 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 2517 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 2518 InOp1.getValueType(), N->getOperand(0), 2519 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 2520 } 2521 2522 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { 2523 assert(N->getValueType(0).isVector() == 2524 N->getOperand(0).getValueType().isVector() && 2525 "Scalar/Vector type mismatch"); 2526 if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N); 2527 2528 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2529 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2530 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2531 return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT, 2532 InOp1, InOp2, N->getOperand(2)); 2533 } 2534 2535 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 2536 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2537 return DAG.getUNDEF(WidenVT); 2538 } 2539 2540 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 2541 EVT VT = N->getValueType(0); 2542 SDLoc dl(N); 2543 2544 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2545 unsigned NumElts = VT.getVectorNumElements(); 2546 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2547 2548 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2549 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2550 2551 // Adjust mask based on new input vector length. 2552 SmallVector<int, 16> NewMask; 2553 for (unsigned i = 0; i != NumElts; ++i) { 2554 int Idx = N->getMaskElt(i); 2555 if (Idx < (int)NumElts) 2556 NewMask.push_back(Idx); 2557 else 2558 NewMask.push_back(Idx - NumElts + WidenNumElts); 2559 } 2560 for (unsigned i = NumElts; i != WidenNumElts; ++i) 2561 NewMask.push_back(-1); 2562 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 2563 } 2564 2565 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 2566 assert(N->getValueType(0).isVector() && 2567 N->getOperand(0).getValueType().isVector() && 2568 "Operands must be vectors"); 2569 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2570 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2571 2572 SDValue InOp1 = N->getOperand(0); 2573 EVT InVT = InOp1.getValueType(); 2574 assert(InVT.isVector() && "can not widen non-vector type"); 2575 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), 2576 InVT.getVectorElementType(), WidenNumElts); 2577 2578 // The input and output types often differ here, and it could be that while 2579 // we'd prefer to widen the result type, the input operands have been split. 2580 // In this case, we also need to split the result of this node as well. 2581 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) { 2582 SDValue SplitVSetCC = SplitVecOp_VSETCC(N); 2583 SDValue Res = ModifyToType(SplitVSetCC, WidenVT); 2584 return Res; 2585 } 2586 2587 InOp1 = GetWidenedVector(InOp1); 2588 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2589 2590 // Assume that the input and output will be widen appropriately. If not, 2591 // we will have to unroll it at some point. 2592 assert(InOp1.getValueType() == WidenInVT && 2593 InOp2.getValueType() == WidenInVT && 2594 "Input not widened to expected type!"); 2595 (void)WidenInVT; 2596 return DAG.getNode(ISD::SETCC, SDLoc(N), 2597 WidenVT, InOp1, InOp2, N->getOperand(2)); 2598 } 2599 2600 2601 //===----------------------------------------------------------------------===// 2602 // Widen Vector Operand 2603 //===----------------------------------------------------------------------===// 2604 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { 2605 DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; 2606 N->dump(&DAG); 2607 dbgs() << "\n"); 2608 SDValue Res = SDValue(); 2609 2610 // See if the target wants to custom widen this node. 2611 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2612 return false; 2613 2614 switch (N->getOpcode()) { 2615 default: 2616 #ifndef NDEBUG 2617 dbgs() << "WidenVectorOperand op #" << OpNo << ": "; 2618 N->dump(&DAG); 2619 dbgs() << "\n"; 2620 #endif 2621 llvm_unreachable("Do not know how to widen this operator's operand!"); 2622 2623 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break; 2624 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 2625 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 2626 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 2627 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 2628 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break; 2629 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; 2630 2631 case ISD::ANY_EXTEND: 2632 case ISD::SIGN_EXTEND: 2633 case ISD::ZERO_EXTEND: 2634 Res = WidenVecOp_EXTEND(N); 2635 break; 2636 2637 case ISD::FP_EXTEND: 2638 case ISD::FP_TO_SINT: 2639 case ISD::FP_TO_UINT: 2640 case ISD::SINT_TO_FP: 2641 case ISD::UINT_TO_FP: 2642 case ISD::TRUNCATE: 2643 Res = WidenVecOp_Convert(N); 2644 break; 2645 } 2646 2647 // If Res is null, the sub-method took care of registering the result. 2648 if (!Res.getNode()) return false; 2649 2650 // If the result is N, the sub-method updated N in place. Tell the legalizer 2651 // core about this. 2652 if (Res.getNode() == N) 2653 return true; 2654 2655 2656 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2657 "Invalid operand expansion"); 2658 2659 ReplaceValueWith(SDValue(N, 0), Res); 2660 return false; 2661 } 2662 2663 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) { 2664 SDLoc DL(N); 2665 EVT VT = N->getValueType(0); 2666 2667 SDValue InOp = N->getOperand(0); 2668 // If some legalization strategy other than widening is used on the operand, 2669 // we can't safely assume that just extending the low lanes is the correct 2670 // transformation. 2671 if (getTypeAction(InOp.getValueType()) != TargetLowering::TypeWidenVector) 2672 return WidenVecOp_Convert(N); 2673 InOp = GetWidenedVector(InOp); 2674 assert(VT.getVectorNumElements() < 2675 InOp.getValueType().getVectorNumElements() && 2676 "Input wasn't widened!"); 2677 2678 // We may need to further widen the operand until it has the same total 2679 // vector size as the result. 2680 EVT InVT = InOp.getValueType(); 2681 if (InVT.getSizeInBits() != VT.getSizeInBits()) { 2682 EVT InEltVT = InVT.getVectorElementType(); 2683 for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) { 2684 EVT FixedVT = (MVT::SimpleValueType)i; 2685 EVT FixedEltVT = FixedVT.getVectorElementType(); 2686 if (TLI.isTypeLegal(FixedVT) && 2687 FixedVT.getSizeInBits() == VT.getSizeInBits() && 2688 FixedEltVT == InEltVT) { 2689 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() && 2690 "Not enough elements in the fixed type for the operand!"); 2691 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() && 2692 "We can't have the same type as we started with!"); 2693 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements()) 2694 InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT, 2695 DAG.getUNDEF(FixedVT), InOp, 2696 DAG.getConstant(0, TLI.getVectorIdxTy())); 2697 else 2698 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp, 2699 DAG.getConstant(0, TLI.getVectorIdxTy())); 2700 break; 2701 } 2702 } 2703 InVT = InOp.getValueType(); 2704 if (InVT.getSizeInBits() != VT.getSizeInBits()) 2705 // We couldn't find a legal vector type that was a widening of the input 2706 // and could be extended in-register to the result type, so we have to 2707 // scalarize. 2708 return WidenVecOp_Convert(N); 2709 } 2710 2711 // Use special DAG nodes to represent the operation of extending the 2712 // low lanes. 2713 switch (N->getOpcode()) { 2714 default: 2715 llvm_unreachable("Extend legalization on on extend operation!"); 2716 case ISD::ANY_EXTEND: 2717 return DAG.getAnyExtendVectorInReg(InOp, DL, VT); 2718 case ISD::SIGN_EXTEND: 2719 return DAG.getSignExtendVectorInReg(InOp, DL, VT); 2720 case ISD::ZERO_EXTEND: 2721 return DAG.getZeroExtendVectorInReg(InOp, DL, VT); 2722 } 2723 } 2724 2725 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 2726 // Since the result is legal and the input is illegal, it is unlikely 2727 // that we can fix the input to a legal type so unroll the convert 2728 // into some scalar code and create a nasty build vector. 2729 EVT VT = N->getValueType(0); 2730 EVT EltVT = VT.getVectorElementType(); 2731 SDLoc dl(N); 2732 unsigned NumElts = VT.getVectorNumElements(); 2733 SDValue InOp = N->getOperand(0); 2734 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2735 InOp = GetWidenedVector(InOp); 2736 EVT InVT = InOp.getValueType(); 2737 EVT InEltVT = InVT.getVectorElementType(); 2738 2739 unsigned Opcode = N->getOpcode(); 2740 SmallVector<SDValue, 16> Ops(NumElts); 2741 for (unsigned i=0; i < NumElts; ++i) 2742 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 2743 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 2744 DAG.getConstant(i, TLI.getVectorIdxTy()))); 2745 2746 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); 2747 } 2748 2749 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { 2750 EVT VT = N->getValueType(0); 2751 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2752 EVT InWidenVT = InOp.getValueType(); 2753 SDLoc dl(N); 2754 2755 // Check if we can convert between two legal vector types and extract. 2756 unsigned InWidenSize = InWidenVT.getSizeInBits(); 2757 unsigned Size = VT.getSizeInBits(); 2758 // x86mmx is not an acceptable vector element type, so don't try. 2759 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) { 2760 unsigned NewNumElts = InWidenSize / Size; 2761 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 2762 if (TLI.isTypeLegal(NewVT)) { 2763 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 2764 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 2765 DAG.getConstant(0, TLI.getVectorIdxTy())); 2766 } 2767 } 2768 2769 return CreateStackStoreLoad(InOp, VT); 2770 } 2771 2772 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 2773 // If the input vector is not legal, it is likely that we will not find a 2774 // legal vector of the same size. Replace the concatenate vector with a 2775 // nasty build vector. 2776 EVT VT = N->getValueType(0); 2777 EVT EltVT = VT.getVectorElementType(); 2778 SDLoc dl(N); 2779 unsigned NumElts = VT.getVectorNumElements(); 2780 SmallVector<SDValue, 16> Ops(NumElts); 2781 2782 EVT InVT = N->getOperand(0).getValueType(); 2783 unsigned NumInElts = InVT.getVectorNumElements(); 2784 2785 unsigned Idx = 0; 2786 unsigned NumOperands = N->getNumOperands(); 2787 for (unsigned i=0; i < NumOperands; ++i) { 2788 SDValue InOp = N->getOperand(i); 2789 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2790 InOp = GetWidenedVector(InOp); 2791 for (unsigned j=0; j < NumInElts; ++j) 2792 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2793 DAG.getConstant(j, TLI.getVectorIdxTy())); 2794 } 2795 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); 2796 } 2797 2798 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 2799 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2800 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), 2801 N->getValueType(0), InOp, N->getOperand(1)); 2802 } 2803 2804 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 2805 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2806 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 2807 N->getValueType(0), InOp, N->getOperand(1)); 2808 } 2809 2810 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 2811 // We have to widen the value but we want only to store the original 2812 // vector type. 2813 StoreSDNode *ST = cast<StoreSDNode>(N); 2814 2815 SmallVector<SDValue, 16> StChain; 2816 if (ST->isTruncatingStore()) 2817 GenWidenVectorTruncStores(StChain, ST); 2818 else 2819 GenWidenVectorStores(StChain, ST); 2820 2821 if (StChain.size() == 1) 2822 return StChain[0]; 2823 else 2824 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain); 2825 } 2826 2827 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) { 2828 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N); 2829 SDValue Mask = MST->getMask(); 2830 EVT MaskVT = Mask.getValueType(); 2831 SDValue StVal = MST->getValue(); 2832 // Widen the value 2833 SDValue WideVal = GetWidenedVector(StVal); 2834 SDLoc dl(N); 2835 2836 if (OpNo == 2 || getTypeAction(MaskVT) == TargetLowering::TypeWidenVector) 2837 Mask = GetWidenedVector(Mask); 2838 else { 2839 // The mask should be widened as well 2840 EVT BoolVT = getSetCCResultType(WideVal.getValueType()); 2841 // We can't use ModifyToType() because we should fill the mask with 2842 // zeroes 2843 unsigned WidenNumElts = BoolVT.getVectorNumElements(); 2844 unsigned MaskNumElts = MaskVT.getVectorNumElements(); 2845 2846 unsigned NumConcat = WidenNumElts / MaskNumElts; 2847 SmallVector<SDValue, 16> Ops(NumConcat); 2848 SDValue ZeroVal = DAG.getConstant(0, MaskVT); 2849 Ops[0] = Mask; 2850 for (unsigned i = 1; i != NumConcat; ++i) 2851 Ops[i] = ZeroVal; 2852 2853 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops); 2854 } 2855 assert(Mask.getValueType().getVectorNumElements() == 2856 WideVal.getValueType().getVectorNumElements() && 2857 "Mask and data vectors should have the same number of elements"); 2858 return DAG.getMaskedStore(MST->getChain(), dl, WideVal, MST->getBasePtr(), 2859 Mask, MST->getMemoryVT(), MST->getMemOperand(), 2860 false); 2861 } 2862 2863 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { 2864 SDValue InOp0 = GetWidenedVector(N->getOperand(0)); 2865 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 2866 SDLoc dl(N); 2867 2868 // WARNING: In this code we widen the compare instruction with garbage. 2869 // This garbage may contain denormal floats which may be slow. Is this a real 2870 // concern ? Should we zero the unused lanes if this is a float compare ? 2871 2872 // Get a new SETCC node to compare the newly widened operands. 2873 // Only some of the compared elements are legal. 2874 EVT SVT = TLI.getSetCCResultType(*DAG.getContext(), InOp0.getValueType()); 2875 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N), 2876 SVT, InOp0, InOp1, N->getOperand(2)); 2877 2878 // Extract the needed results from the result vector. 2879 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), 2880 SVT.getVectorElementType(), 2881 N->getValueType(0).getVectorNumElements()); 2882 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, 2883 ResVT, WideSETCC, DAG.getConstant(0, 2884 TLI.getVectorIdxTy())); 2885 2886 return PromoteTargetBoolean(CC, N->getValueType(0)); 2887 } 2888 2889 2890 //===----------------------------------------------------------------------===// 2891 // Vector Widening Utilities 2892 //===----------------------------------------------------------------------===// 2893 2894 // Utility function to find the type to chop up a widen vector for load/store 2895 // TLI: Target lowering used to determine legal types. 2896 // Width: Width left need to load/store. 2897 // WidenVT: The widen vector type to load to/store from 2898 // Align: If 0, don't allow use of a wider type 2899 // WidenEx: If Align is not 0, the amount additional we can load/store from. 2900 2901 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI, 2902 unsigned Width, EVT WidenVT, 2903 unsigned Align = 0, unsigned WidenEx = 0) { 2904 EVT WidenEltVT = WidenVT.getVectorElementType(); 2905 unsigned WidenWidth = WidenVT.getSizeInBits(); 2906 unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); 2907 unsigned AlignInBits = Align*8; 2908 2909 // If we have one element to load/store, return it. 2910 EVT RetVT = WidenEltVT; 2911 if (Width == WidenEltWidth) 2912 return RetVT; 2913 2914 // See if there is larger legal integer than the element type to load/store 2915 unsigned VT; 2916 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE; 2917 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) { 2918 EVT MemVT((MVT::SimpleValueType) VT); 2919 unsigned MemVTWidth = MemVT.getSizeInBits(); 2920 if (MemVT.getSizeInBits() <= WidenEltWidth) 2921 break; 2922 if (TLI.isTypeLegal(MemVT) && (WidenWidth % MemVTWidth) == 0 && 2923 isPowerOf2_32(WidenWidth / MemVTWidth) && 2924 (MemVTWidth <= Width || 2925 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2926 RetVT = MemVT; 2927 break; 2928 } 2929 } 2930 2931 // See if there is a larger vector type to load/store that has the same vector 2932 // element type and is evenly divisible with the WidenVT. 2933 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE; 2934 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) { 2935 EVT MemVT = (MVT::SimpleValueType) VT; 2936 unsigned MemVTWidth = MemVT.getSizeInBits(); 2937 if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() && 2938 (WidenWidth % MemVTWidth) == 0 && 2939 isPowerOf2_32(WidenWidth / MemVTWidth) && 2940 (MemVTWidth <= Width || 2941 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2942 if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT) 2943 return MemVT; 2944 } 2945 } 2946 2947 return RetVT; 2948 } 2949 2950 // Builds a vector type from scalar loads 2951 // VecTy: Resulting Vector type 2952 // LDOps: Load operators to build a vector type 2953 // [Start,End) the list of loads to use. 2954 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, 2955 SmallVectorImpl<SDValue> &LdOps, 2956 unsigned Start, unsigned End) { 2957 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2958 SDLoc dl(LdOps[Start]); 2959 EVT LdTy = LdOps[Start].getValueType(); 2960 unsigned Width = VecTy.getSizeInBits(); 2961 unsigned NumElts = Width / LdTy.getSizeInBits(); 2962 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts); 2963 2964 unsigned Idx = 1; 2965 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]); 2966 2967 for (unsigned i = Start + 1; i != End; ++i) { 2968 EVT NewLdTy = LdOps[i].getValueType(); 2969 if (NewLdTy != LdTy) { 2970 NumElts = Width / NewLdTy.getSizeInBits(); 2971 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts); 2972 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp); 2973 // Readjust position and vector position based on new load type 2974 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); 2975 LdTy = NewLdTy; 2976 } 2977 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i], 2978 DAG.getConstant(Idx++, TLI.getVectorIdxTy())); 2979 } 2980 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp); 2981 } 2982 2983 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain, 2984 LoadSDNode *LD) { 2985 // The strategy assumes that we can efficiently load powers of two widths. 2986 // The routines chops the vector into the largest vector loads with the same 2987 // element type or scalar loads and then recombines it to the widen vector 2988 // type. 2989 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2990 unsigned WidenWidth = WidenVT.getSizeInBits(); 2991 EVT LdVT = LD->getMemoryVT(); 2992 SDLoc dl(LD); 2993 assert(LdVT.isVector() && WidenVT.isVector()); 2994 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 2995 2996 // Load information 2997 SDValue Chain = LD->getChain(); 2998 SDValue BasePtr = LD->getBasePtr(); 2999 unsigned Align = LD->getAlignment(); 3000 bool isVolatile = LD->isVolatile(); 3001 bool isNonTemporal = LD->isNonTemporal(); 3002 bool isInvariant = LD->isInvariant(); 3003 AAMDNodes AAInfo = LD->getAAInfo(); 3004 3005 int LdWidth = LdVT.getSizeInBits(); 3006 int WidthDiff = WidenWidth - LdWidth; // Difference 3007 unsigned LdAlign = (isVolatile) ? 0 : Align; // Allow wider loads 3008 3009 // Find the vector type that can load from. 3010 EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 3011 int NewVTWidth = NewVT.getSizeInBits(); 3012 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(), 3013 isVolatile, isNonTemporal, isInvariant, Align, 3014 AAInfo); 3015 LdChain.push_back(LdOp.getValue(1)); 3016 3017 // Check if we can load the element with one instruction 3018 if (LdWidth <= NewVTWidth) { 3019 if (!NewVT.isVector()) { 3020 unsigned NumElts = WidenWidth / NewVTWidth; 3021 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 3022 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 3023 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp); 3024 } 3025 if (NewVT == WidenVT) 3026 return LdOp; 3027 3028 assert(WidenWidth % NewVTWidth == 0); 3029 unsigned NumConcat = WidenWidth / NewVTWidth; 3030 SmallVector<SDValue, 16> ConcatOps(NumConcat); 3031 SDValue UndefVal = DAG.getUNDEF(NewVT); 3032 ConcatOps[0] = LdOp; 3033 for (unsigned i = 1; i != NumConcat; ++i) 3034 ConcatOps[i] = UndefVal; 3035 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps); 3036 } 3037 3038 // Load vector by using multiple loads from largest vector to scalar 3039 SmallVector<SDValue, 16> LdOps; 3040 LdOps.push_back(LdOp); 3041 3042 LdWidth -= NewVTWidth; 3043 unsigned Offset = 0; 3044 3045 while (LdWidth > 0) { 3046 unsigned Increment = NewVTWidth / 8; 3047 Offset += Increment; 3048 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 3049 DAG.getConstant(Increment, BasePtr.getValueType())); 3050 3051 SDValue L; 3052 if (LdWidth < NewVTWidth) { 3053 // Our current type we are using is too large, find a better size 3054 NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 3055 NewVTWidth = NewVT.getSizeInBits(); 3056 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 3057 LD->getPointerInfo().getWithOffset(Offset), isVolatile, 3058 isNonTemporal, isInvariant, MinAlign(Align, Increment), 3059 AAInfo); 3060 LdChain.push_back(L.getValue(1)); 3061 if (L->getValueType(0).isVector()) { 3062 SmallVector<SDValue, 16> Loads; 3063 Loads.push_back(L); 3064 unsigned size = L->getValueSizeInBits(0); 3065 while (size < LdOp->getValueSizeInBits(0)) { 3066 Loads.push_back(DAG.getUNDEF(L->getValueType(0))); 3067 size += L->getValueSizeInBits(0); 3068 } 3069 L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads); 3070 } 3071 } else { 3072 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 3073 LD->getPointerInfo().getWithOffset(Offset), isVolatile, 3074 isNonTemporal, isInvariant, MinAlign(Align, Increment), 3075 AAInfo); 3076 LdChain.push_back(L.getValue(1)); 3077 } 3078 3079 LdOps.push_back(L); 3080 3081 3082 LdWidth -= NewVTWidth; 3083 } 3084 3085 // Build the vector from the loads operations 3086 unsigned End = LdOps.size(); 3087 if (!LdOps[0].getValueType().isVector()) 3088 // All the loads are scalar loads. 3089 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End); 3090 3091 // If the load contains vectors, build the vector using concat vector. 3092 // All of the vectors used to loads are power of 2 and the scalars load 3093 // can be combined to make a power of 2 vector. 3094 SmallVector<SDValue, 16> ConcatOps(End); 3095 int i = End - 1; 3096 int Idx = End; 3097 EVT LdTy = LdOps[i].getValueType(); 3098 // First combine the scalar loads to a vector 3099 if (!LdTy.isVector()) { 3100 for (--i; i >= 0; --i) { 3101 LdTy = LdOps[i].getValueType(); 3102 if (LdTy.isVector()) 3103 break; 3104 } 3105 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i+1, End); 3106 } 3107 ConcatOps[--Idx] = LdOps[i]; 3108 for (--i; i >= 0; --i) { 3109 EVT NewLdTy = LdOps[i].getValueType(); 3110 if (NewLdTy != LdTy) { 3111 // Create a larger vector 3112 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy, 3113 makeArrayRef(&ConcatOps[Idx], End - Idx)); 3114 Idx = End - 1; 3115 LdTy = NewLdTy; 3116 } 3117 ConcatOps[--Idx] = LdOps[i]; 3118 } 3119 3120 if (WidenWidth == LdTy.getSizeInBits()*(End - Idx)) 3121 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 3122 makeArrayRef(&ConcatOps[Idx], End - Idx)); 3123 3124 // We need to fill the rest with undefs to build the vector 3125 unsigned NumOps = WidenWidth / LdTy.getSizeInBits(); 3126 SmallVector<SDValue, 16> WidenOps(NumOps); 3127 SDValue UndefVal = DAG.getUNDEF(LdTy); 3128 { 3129 unsigned i = 0; 3130 for (; i != End-Idx; ++i) 3131 WidenOps[i] = ConcatOps[Idx+i]; 3132 for (; i != NumOps; ++i) 3133 WidenOps[i] = UndefVal; 3134 } 3135 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps); 3136 } 3137 3138 SDValue 3139 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain, 3140 LoadSDNode *LD, 3141 ISD::LoadExtType ExtType) { 3142 // For extension loads, it may not be more efficient to chop up the vector 3143 // and then extended it. Instead, we unroll the load and build a new vector. 3144 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 3145 EVT LdVT = LD->getMemoryVT(); 3146 SDLoc dl(LD); 3147 assert(LdVT.isVector() && WidenVT.isVector()); 3148 3149 // Load information 3150 SDValue Chain = LD->getChain(); 3151 SDValue BasePtr = LD->getBasePtr(); 3152 unsigned Align = LD->getAlignment(); 3153 bool isVolatile = LD->isVolatile(); 3154 bool isNonTemporal = LD->isNonTemporal(); 3155 bool isInvariant = LD->isInvariant(); 3156 AAMDNodes AAInfo = LD->getAAInfo(); 3157 3158 EVT EltVT = WidenVT.getVectorElementType(); 3159 EVT LdEltVT = LdVT.getVectorElementType(); 3160 unsigned NumElts = LdVT.getVectorNumElements(); 3161 3162 // Load each element and widen 3163 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3164 SmallVector<SDValue, 16> Ops(WidenNumElts); 3165 unsigned Increment = LdEltVT.getSizeInBits() / 8; 3166 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, 3167 LD->getPointerInfo(), 3168 LdEltVT, isVolatile, isNonTemporal, isInvariant, 3169 Align, AAInfo); 3170 LdChain.push_back(Ops[0].getValue(1)); 3171 unsigned i = 0, Offset = Increment; 3172 for (i=1; i < NumElts; ++i, Offset += Increment) { 3173 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 3174 BasePtr, 3175 DAG.getConstant(Offset, 3176 BasePtr.getValueType())); 3177 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, 3178 LD->getPointerInfo().getWithOffset(Offset), LdEltVT, 3179 isVolatile, isNonTemporal, isInvariant, Align, 3180 AAInfo); 3181 LdChain.push_back(Ops[i].getValue(1)); 3182 } 3183 3184 // Fill the rest with undefs 3185 SDValue UndefVal = DAG.getUNDEF(EltVT); 3186 for (; i != WidenNumElts; ++i) 3187 Ops[i] = UndefVal; 3188 3189 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 3190 } 3191 3192 3193 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, 3194 StoreSDNode *ST) { 3195 // The strategy assumes that we can efficiently store powers of two widths. 3196 // The routines chops the vector into the largest vector stores with the same 3197 // element type or scalar stores. 3198 SDValue Chain = ST->getChain(); 3199 SDValue BasePtr = ST->getBasePtr(); 3200 unsigned Align = ST->getAlignment(); 3201 bool isVolatile = ST->isVolatile(); 3202 bool isNonTemporal = ST->isNonTemporal(); 3203 AAMDNodes AAInfo = ST->getAAInfo(); 3204 SDValue ValOp = GetWidenedVector(ST->getValue()); 3205 SDLoc dl(ST); 3206 3207 EVT StVT = ST->getMemoryVT(); 3208 unsigned StWidth = StVT.getSizeInBits(); 3209 EVT ValVT = ValOp.getValueType(); 3210 unsigned ValWidth = ValVT.getSizeInBits(); 3211 EVT ValEltVT = ValVT.getVectorElementType(); 3212 unsigned ValEltWidth = ValEltVT.getSizeInBits(); 3213 assert(StVT.getVectorElementType() == ValEltVT); 3214 3215 int Idx = 0; // current index to store 3216 unsigned Offset = 0; // offset from base to store 3217 while (StWidth != 0) { 3218 // Find the largest vector type we can store with 3219 EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT); 3220 unsigned NewVTWidth = NewVT.getSizeInBits(); 3221 unsigned Increment = NewVTWidth / 8; 3222 if (NewVT.isVector()) { 3223 unsigned NumVTElts = NewVT.getVectorNumElements(); 3224 do { 3225 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp, 3226 DAG.getConstant(Idx, TLI.getVectorIdxTy())); 3227 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 3228 ST->getPointerInfo().getWithOffset(Offset), 3229 isVolatile, isNonTemporal, 3230 MinAlign(Align, Offset), AAInfo)); 3231 StWidth -= NewVTWidth; 3232 Offset += Increment; 3233 Idx += NumVTElts; 3234 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 3235 DAG.getConstant(Increment, BasePtr.getValueType())); 3236 } while (StWidth != 0 && StWidth >= NewVTWidth); 3237 } else { 3238 // Cast the vector to the scalar type we can store 3239 unsigned NumElts = ValWidth / NewVTWidth; 3240 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 3241 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp); 3242 // Readjust index position based on new vector type 3243 Idx = Idx * ValEltWidth / NewVTWidth; 3244 do { 3245 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp, 3246 DAG.getConstant(Idx++, TLI.getVectorIdxTy())); 3247 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 3248 ST->getPointerInfo().getWithOffset(Offset), 3249 isVolatile, isNonTemporal, 3250 MinAlign(Align, Offset), AAInfo)); 3251 StWidth -= NewVTWidth; 3252 Offset += Increment; 3253 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 3254 DAG.getConstant(Increment, BasePtr.getValueType())); 3255 } while (StWidth != 0 && StWidth >= NewVTWidth); 3256 // Restore index back to be relative to the original widen element type 3257 Idx = Idx * NewVTWidth / ValEltWidth; 3258 } 3259 } 3260 } 3261 3262 void 3263 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain, 3264 StoreSDNode *ST) { 3265 // For extension loads, it may not be more efficient to truncate the vector 3266 // and then store it. Instead, we extract each element and then store it. 3267 SDValue Chain = ST->getChain(); 3268 SDValue BasePtr = ST->getBasePtr(); 3269 unsigned Align = ST->getAlignment(); 3270 bool isVolatile = ST->isVolatile(); 3271 bool isNonTemporal = ST->isNonTemporal(); 3272 AAMDNodes AAInfo = ST->getAAInfo(); 3273 SDValue ValOp = GetWidenedVector(ST->getValue()); 3274 SDLoc dl(ST); 3275 3276 EVT StVT = ST->getMemoryVT(); 3277 EVT ValVT = ValOp.getValueType(); 3278 3279 // It must be true that we the widen vector type is bigger than where 3280 // we need to store. 3281 assert(StVT.isVector() && ValOp.getValueType().isVector()); 3282 assert(StVT.bitsLT(ValOp.getValueType())); 3283 3284 // For truncating stores, we can not play the tricks of chopping legal 3285 // vector types and bit cast it to the right type. Instead, we unroll 3286 // the store. 3287 EVT StEltVT = StVT.getVectorElementType(); 3288 EVT ValEltVT = ValVT.getVectorElementType(); 3289 unsigned Increment = ValEltVT.getSizeInBits() / 8; 3290 unsigned NumElts = StVT.getVectorNumElements(); 3291 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 3292 DAG.getConstant(0, TLI.getVectorIdxTy())); 3293 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, 3294 ST->getPointerInfo(), StEltVT, 3295 isVolatile, isNonTemporal, Align, 3296 AAInfo)); 3297 unsigned Offset = Increment; 3298 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 3299 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 3300 BasePtr, DAG.getConstant(Offset, 3301 BasePtr.getValueType())); 3302 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 3303 DAG.getConstant(0, TLI.getVectorIdxTy())); 3304 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, 3305 ST->getPointerInfo().getWithOffset(Offset), 3306 StEltVT, isVolatile, isNonTemporal, 3307 MinAlign(Align, Offset), AAInfo)); 3308 } 3309 } 3310 3311 /// Modifies a vector input (widen or narrows) to a vector of NVT. The 3312 /// input vector must have the same element type as NVT. 3313 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) { 3314 // Note that InOp might have been widened so it might already have 3315 // the right width or it might need be narrowed. 3316 EVT InVT = InOp.getValueType(); 3317 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 3318 "input and widen element type must match"); 3319 SDLoc dl(InOp); 3320 3321 // Check if InOp already has the right width. 3322 if (InVT == NVT) 3323 return InOp; 3324 3325 unsigned InNumElts = InVT.getVectorNumElements(); 3326 unsigned WidenNumElts = NVT.getVectorNumElements(); 3327 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 3328 unsigned NumConcat = WidenNumElts / InNumElts; 3329 SmallVector<SDValue, 16> Ops(NumConcat); 3330 SDValue UndefVal = DAG.getUNDEF(InVT); 3331 Ops[0] = InOp; 3332 for (unsigned i = 1; i != NumConcat; ++i) 3333 Ops[i] = UndefVal; 3334 3335 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops); 3336 } 3337 3338 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 3339 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 3340 DAG.getConstant(0, TLI.getVectorIdxTy())); 3341 3342 // Fall back to extract and build. 3343 SmallVector<SDValue, 16> Ops(WidenNumElts); 3344 EVT EltVT = NVT.getVectorElementType(); 3345 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 3346 unsigned Idx; 3347 for (Idx = 0; Idx < MinNumElts; ++Idx) 3348 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 3349 DAG.getConstant(Idx, TLI.getVectorIdxTy())); 3350 3351 SDValue UndefVal = DAG.getUNDEF(EltVT); 3352 for ( ; Idx < WidenNumElts; ++Idx) 3353 Ops[Idx] = UndefVal; 3354 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, Ops); 3355 } 3356