1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file performs vector type splitting and scalarization for LegalizeTypes. 11 // Scalarization is the act of changing a computation in an illegal one-element 12 // vector type to be a computation in its scalar element type. For example, 13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which 15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16 // types. 17 // Splitting is the act of changing a computation in an invalid vector type to 18 // be a computation in two vectors of half the size. For example, implementing 19 // <128 x f32> operations in terms of two <64 x f32> operations. 20 // 21 //===----------------------------------------------------------------------===// 22 23 #include "LegalizeTypes.h" 24 #include "llvm/IR/DataLayout.h" 25 #include "llvm/Support/ErrorHandling.h" 26 #include "llvm/Support/raw_ostream.h" 27 using namespace llvm; 28 29 #define DEBUG_TYPE "legalize-types" 30 31 //===----------------------------------------------------------------------===// 32 // Result Vector Scalarization: <1 x ty> -> ty. 33 //===----------------------------------------------------------------------===// 34 35 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 36 DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; 37 N->dump(&DAG); 38 dbgs() << "\n"); 39 SDValue R = SDValue(); 40 41 switch (N->getOpcode()) { 42 default: 43 #ifndef NDEBUG 44 dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 45 N->dump(&DAG); 46 dbgs() << "\n"; 47 #endif 48 report_fatal_error("Do not know how to scalarize the result of this " 49 "operator!\n"); 50 51 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; 52 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; 53 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; 54 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 55 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 56 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; 57 case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break; 58 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 59 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 60 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 61 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 62 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 63 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break; 64 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 65 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 66 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 67 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 68 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 69 case ISD::ANY_EXTEND: 70 case ISD::BITREVERSE: 71 case ISD::BSWAP: 72 case ISD::CTLZ: 73 case ISD::CTLZ_ZERO_UNDEF: 74 case ISD::CTPOP: 75 case ISD::CTTZ: 76 case ISD::CTTZ_ZERO_UNDEF: 77 case ISD::FABS: 78 case ISD::FCEIL: 79 case ISD::FCOS: 80 case ISD::FEXP: 81 case ISD::FEXP2: 82 case ISD::FFLOOR: 83 case ISD::FLOG: 84 case ISD::FLOG10: 85 case ISD::FLOG2: 86 case ISD::FNEARBYINT: 87 case ISD::FNEG: 88 case ISD::FP_EXTEND: 89 case ISD::FP_TO_SINT: 90 case ISD::FP_TO_UINT: 91 case ISD::FRINT: 92 case ISD::FROUND: 93 case ISD::FSIN: 94 case ISD::FSQRT: 95 case ISD::FTRUNC: 96 case ISD::SIGN_EXTEND: 97 case ISD::SINT_TO_FP: 98 case ISD::TRUNCATE: 99 case ISD::UINT_TO_FP: 100 case ISD::ZERO_EXTEND: 101 R = ScalarizeVecRes_UnaryOp(N); 102 break; 103 104 case ISD::ADD: 105 case ISD::AND: 106 case ISD::FADD: 107 case ISD::FCOPYSIGN: 108 case ISD::FDIV: 109 case ISD::FMUL: 110 case ISD::FMINNUM: 111 case ISD::FMAXNUM: 112 case ISD::FMINNAN: 113 case ISD::FMAXNAN: 114 case ISD::SMIN: 115 case ISD::SMAX: 116 case ISD::UMIN: 117 case ISD::UMAX: 118 119 case ISD::FPOW: 120 case ISD::FREM: 121 case ISD::FSUB: 122 case ISD::MUL: 123 case ISD::OR: 124 case ISD::SDIV: 125 case ISD::SREM: 126 case ISD::SUB: 127 case ISD::UDIV: 128 case ISD::UREM: 129 case ISD::XOR: 130 case ISD::SHL: 131 case ISD::SRA: 132 case ISD::SRL: 133 R = ScalarizeVecRes_BinOp(N); 134 break; 135 case ISD::FMA: 136 R = ScalarizeVecRes_TernaryOp(N); 137 break; 138 } 139 140 // If R is null, the sub-method took care of registering the result. 141 if (R.getNode()) 142 SetScalarizedVector(SDValue(N, ResNo), R); 143 } 144 145 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 146 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 147 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 148 return DAG.getNode(N->getOpcode(), SDLoc(N), 149 LHS.getValueType(), LHS, RHS, N->getFlags()); 150 } 151 152 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { 153 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 154 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 155 SDValue Op2 = GetScalarizedVector(N->getOperand(2)); 156 return DAG.getNode(N->getOpcode(), SDLoc(N), 157 Op0.getValueType(), Op0, Op1, Op2); 158 } 159 160 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, 161 unsigned ResNo) { 162 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 163 return GetScalarizedVector(Op); 164 } 165 166 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { 167 EVT NewVT = N->getValueType(0).getVectorElementType(); 168 return DAG.getNode(ISD::BITCAST, SDLoc(N), 169 NewVT, N->getOperand(0)); 170 } 171 172 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { 173 EVT EltVT = N->getValueType(0).getVectorElementType(); 174 SDValue InOp = N->getOperand(0); 175 // The BUILD_VECTOR operands may be of wider element types and 176 // we may need to truncate them back to the requested return type. 177 if (EltVT.isInteger()) 178 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 179 return InOp; 180 } 181 182 SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 183 EVT NewVT = N->getValueType(0).getVectorElementType(); 184 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 185 return DAG.getConvertRndSat(NewVT, SDLoc(N), 186 Op0, DAG.getValueType(NewVT), 187 DAG.getValueType(Op0.getValueType()), 188 N->getOperand(3), 189 N->getOperand(4), 190 cast<CvtRndSatSDNode>(N)->getCvtCode()); 191 } 192 193 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 194 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 195 N->getValueType(0).getVectorElementType(), 196 N->getOperand(0), N->getOperand(1)); 197 } 198 199 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { 200 EVT NewVT = N->getValueType(0).getVectorElementType(); 201 SDValue Op = GetScalarizedVector(N->getOperand(0)); 202 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 203 NewVT, Op, N->getOperand(1)); 204 } 205 206 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 207 SDValue Op = GetScalarizedVector(N->getOperand(0)); 208 return DAG.getNode(ISD::FPOWI, SDLoc(N), 209 Op.getValueType(), Op, N->getOperand(1)); 210 } 211 212 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 213 // The value to insert may have a wider type than the vector element type, 214 // so be sure to truncate it to the element type if necessary. 215 SDValue Op = N->getOperand(1); 216 EVT EltVT = N->getValueType(0).getVectorElementType(); 217 if (Op.getValueType() != EltVT) 218 // FIXME: Can this happen for floating point types? 219 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op); 220 return Op; 221 } 222 223 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 224 assert(N->isUnindexed() && "Indexed vector load?"); 225 226 SDValue Result = DAG.getLoad(ISD::UNINDEXED, 227 N->getExtensionType(), 228 N->getValueType(0).getVectorElementType(), 229 SDLoc(N), 230 N->getChain(), N->getBasePtr(), 231 DAG.getUNDEF(N->getBasePtr().getValueType()), 232 N->getPointerInfo(), 233 N->getMemoryVT().getVectorElementType(), 234 N->isVolatile(), N->isNonTemporal(), 235 N->isInvariant(), N->getOriginalAlignment(), 236 N->getAAInfo()); 237 238 // Legalize the chain result - switch anything that used the old chain to 239 // use the new one. 240 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 241 return Result; 242 } 243 244 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 245 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 246 EVT DestVT = N->getValueType(0).getVectorElementType(); 247 SDValue Op = N->getOperand(0); 248 EVT OpVT = Op.getValueType(); 249 SDLoc DL(N); 250 // The result needs scalarizing, but it's not a given that the source does. 251 // This is a workaround for targets where it's impossible to scalarize the 252 // result of a conversion, because the source type is legal. 253 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32} 254 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is 255 // legal and was not scalarized. 256 // See the similar logic in ScalarizeVecRes_VSETCC 257 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 258 Op = GetScalarizedVector(Op); 259 } else { 260 EVT VT = OpVT.getVectorElementType(); 261 Op = DAG.getNode( 262 ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, 263 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 264 } 265 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op); 266 } 267 268 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 269 EVT EltVT = N->getValueType(0).getVectorElementType(); 270 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 271 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 272 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT, 273 LHS, DAG.getValueType(ExtVT)); 274 } 275 276 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 277 // If the operand is wider than the vector element type then it is implicitly 278 // truncated. Make that explicit here. 279 EVT EltVT = N->getValueType(0).getVectorElementType(); 280 SDValue InOp = N->getOperand(0); 281 if (InOp.getValueType() != EltVT) 282 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 283 return InOp; 284 } 285 286 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { 287 SDValue Cond = GetScalarizedVector(N->getOperand(0)); 288 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 289 TargetLowering::BooleanContent ScalarBool = 290 TLI.getBooleanContents(false, false); 291 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false); 292 293 // If integer and float booleans have different contents then we can't 294 // reliably optimize in all cases. There is a full explanation for this in 295 // DAGCombiner::visitSELECT() where the same issue affects folding 296 // (select C, 0, 1) to (xor C, 1). 297 if (TLI.getBooleanContents(false, false) != 298 TLI.getBooleanContents(false, true)) { 299 // At least try the common case where the boolean is generated by a 300 // comparison. 301 if (Cond->getOpcode() == ISD::SETCC) { 302 EVT OpVT = Cond->getOperand(0)->getValueType(0); 303 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType()); 304 VecBool = TLI.getBooleanContents(OpVT); 305 } else 306 ScalarBool = TargetLowering::UndefinedBooleanContent; 307 } 308 309 if (ScalarBool != VecBool) { 310 EVT CondVT = Cond.getValueType(); 311 switch (ScalarBool) { 312 case TargetLowering::UndefinedBooleanContent: 313 break; 314 case TargetLowering::ZeroOrOneBooleanContent: 315 assert(VecBool == TargetLowering::UndefinedBooleanContent || 316 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent); 317 // Vector read from all ones, scalar expects a single 1 so mask. 318 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT, 319 Cond, DAG.getConstant(1, SDLoc(N), CondVT)); 320 break; 321 case TargetLowering::ZeroOrNegativeOneBooleanContent: 322 assert(VecBool == TargetLowering::UndefinedBooleanContent || 323 VecBool == TargetLowering::ZeroOrOneBooleanContent); 324 // Vector reads from a one, scalar from all ones so sign extend. 325 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT, 326 Cond, DAG.getValueType(MVT::i1)); 327 break; 328 } 329 } 330 331 return DAG.getSelect(SDLoc(N), 332 LHS.getValueType(), Cond, LHS, 333 GetScalarizedVector(N->getOperand(2))); 334 } 335 336 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 337 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 338 return DAG.getSelect(SDLoc(N), 339 LHS.getValueType(), N->getOperand(0), LHS, 340 GetScalarizedVector(N->getOperand(2))); 341 } 342 343 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 344 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 345 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(), 346 N->getOperand(0), N->getOperand(1), 347 LHS, GetScalarizedVector(N->getOperand(3)), 348 N->getOperand(4)); 349 } 350 351 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 352 assert(N->getValueType(0).isVector() == 353 N->getOperand(0).getValueType().isVector() && 354 "Scalar/Vector type mismatch"); 355 356 if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N); 357 358 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 359 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 360 SDLoc DL(N); 361 362 // Turn it into a scalar SETCC. 363 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 364 } 365 366 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 367 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 368 } 369 370 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 371 // Figure out if the scalar is the LHS or RHS and return it. 372 SDValue Arg = N->getOperand(2).getOperand(0); 373 if (Arg.isUndef()) 374 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 375 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 376 return GetScalarizedVector(N->getOperand(Op)); 377 } 378 379 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 380 assert(N->getValueType(0).isVector() && 381 N->getOperand(0).getValueType().isVector() && 382 "Operand types must be vectors"); 383 SDValue LHS = N->getOperand(0); 384 SDValue RHS = N->getOperand(1); 385 EVT OpVT = LHS.getValueType(); 386 EVT NVT = N->getValueType(0).getVectorElementType(); 387 SDLoc DL(N); 388 389 // The result needs scalarizing, but it's not a given that the source does. 390 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 391 LHS = GetScalarizedVector(LHS); 392 RHS = GetScalarizedVector(RHS); 393 } else { 394 EVT VT = OpVT.getVectorElementType(); 395 LHS = DAG.getNode( 396 ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS, 397 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 398 RHS = DAG.getNode( 399 ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS, 400 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 401 } 402 403 // Turn it into a scalar SETCC. 404 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 405 N->getOperand(2)); 406 // Vectors may have a different boolean contents to scalars. Promote the 407 // value appropriately. 408 ISD::NodeType ExtendCode = 409 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 410 return DAG.getNode(ExtendCode, DL, NVT, Res); 411 } 412 413 414 //===----------------------------------------------------------------------===// 415 // Operand Vector Scalarization <1 x ty> -> ty. 416 //===----------------------------------------------------------------------===// 417 418 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 419 DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; 420 N->dump(&DAG); 421 dbgs() << "\n"); 422 SDValue Res = SDValue(); 423 424 if (!Res.getNode()) { 425 switch (N->getOpcode()) { 426 default: 427 #ifndef NDEBUG 428 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 429 N->dump(&DAG); 430 dbgs() << "\n"; 431 #endif 432 llvm_unreachable("Do not know how to scalarize this operator's operand!"); 433 case ISD::BITCAST: 434 Res = ScalarizeVecOp_BITCAST(N); 435 break; 436 case ISD::ANY_EXTEND: 437 case ISD::ZERO_EXTEND: 438 case ISD::SIGN_EXTEND: 439 case ISD::TRUNCATE: 440 case ISD::FP_TO_SINT: 441 case ISD::FP_TO_UINT: 442 case ISD::SINT_TO_FP: 443 case ISD::UINT_TO_FP: 444 Res = ScalarizeVecOp_UnaryOp(N); 445 break; 446 case ISD::CONCAT_VECTORS: 447 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 448 break; 449 case ISD::EXTRACT_VECTOR_ELT: 450 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 451 break; 452 case ISD::VSELECT: 453 Res = ScalarizeVecOp_VSELECT(N); 454 break; 455 case ISD::STORE: 456 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 457 break; 458 case ISD::FP_ROUND: 459 Res = ScalarizeVecOp_FP_ROUND(N, OpNo); 460 break; 461 } 462 } 463 464 // If the result is null, the sub-method took care of registering results etc. 465 if (!Res.getNode()) return false; 466 467 // If the result is N, the sub-method updated N in place. Tell the legalizer 468 // core about this. 469 if (Res.getNode() == N) 470 return true; 471 472 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 473 "Invalid operand expansion"); 474 475 ReplaceValueWith(SDValue(N, 0), Res); 476 return false; 477 } 478 479 /// If the value to convert is a vector that needs to be scalarized, it must be 480 /// <1 x ty>. Convert the element instead. 481 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { 482 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 483 return DAG.getNode(ISD::BITCAST, SDLoc(N), 484 N->getValueType(0), Elt); 485 } 486 487 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>. 488 /// Do the operation on the element instead. 489 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) { 490 assert(N->getValueType(0).getVectorNumElements() == 1 && 491 "Unexpected vector type!"); 492 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 493 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N), 494 N->getValueType(0).getScalarType(), Elt); 495 // Revectorize the result so the types line up with what the uses of this 496 // expression expect. 497 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), Op); 498 } 499 500 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead. 501 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 502 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 503 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 504 Ops[i] = GetScalarizedVector(N->getOperand(i)); 505 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0), Ops); 506 } 507 508 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>, 509 /// so just return the element, ignoring the index. 510 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 511 SDValue Res = GetScalarizedVector(N->getOperand(0)); 512 if (Res.getValueType() != N->getValueType(0)) 513 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), 514 Res); 515 return Res; 516 } 517 518 519 /// If the input condition is a vector that needs to be scalarized, it must be 520 /// <1 x i1>, so just convert to a normal ISD::SELECT 521 /// (still with vector output type since that was acceptable if we got here). 522 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) { 523 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0)); 524 EVT VT = N->getValueType(0); 525 526 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1), 527 N->getOperand(2)); 528 } 529 530 /// If the value to store is a vector that needs to be scalarized, it must be 531 /// <1 x ty>. Just store the element. 532 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 533 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 534 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 535 SDLoc dl(N); 536 537 if (N->isTruncatingStore()) 538 return DAG.getTruncStore(N->getChain(), dl, 539 GetScalarizedVector(N->getOperand(1)), 540 N->getBasePtr(), N->getPointerInfo(), 541 N->getMemoryVT().getVectorElementType(), 542 N->isVolatile(), N->isNonTemporal(), 543 N->getAlignment(), N->getAAInfo()); 544 545 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 546 N->getBasePtr(), N->getPointerInfo(), 547 N->isVolatile(), N->isNonTemporal(), 548 N->getOriginalAlignment(), N->getAAInfo()); 549 } 550 551 /// If the value to round is a vector that needs to be scalarized, it must be 552 /// <1 x ty>. Convert the element instead. 553 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) { 554 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 555 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N), 556 N->getValueType(0).getVectorElementType(), Elt, 557 N->getOperand(1)); 558 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 559 } 560 561 //===----------------------------------------------------------------------===// 562 // Result Vector Splitting 563 //===----------------------------------------------------------------------===// 564 565 /// This method is called when the specified result of the specified node is 566 /// found to need vector splitting. At this point, the node may also have 567 /// invalid operands or may have other results that need legalization, we just 568 /// know that (at least) one result needs vector splitting. 569 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 570 DEBUG(dbgs() << "Split node result: "; 571 N->dump(&DAG); 572 dbgs() << "\n"); 573 SDValue Lo, Hi; 574 575 // See if the target wants to custom expand this node. 576 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 577 return; 578 579 switch (N->getOpcode()) { 580 default: 581 #ifndef NDEBUG 582 dbgs() << "SplitVectorResult #" << ResNo << ": "; 583 N->dump(&DAG); 584 dbgs() << "\n"; 585 #endif 586 report_fatal_error("Do not know how to split the result of this " 587 "operator!\n"); 588 589 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 590 case ISD::VSELECT: 591 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 592 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 593 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 594 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break; 595 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 596 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 597 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 598 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break; 599 case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 600 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 601 case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break; 602 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 603 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 604 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 605 case ISD::LOAD: 606 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 607 break; 608 case ISD::MLOAD: 609 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi); 610 break; 611 case ISD::MGATHER: 612 SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi); 613 break; 614 case ISD::SETCC: 615 SplitVecRes_SETCC(N, Lo, Hi); 616 break; 617 case ISD::VECTOR_SHUFFLE: 618 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 619 break; 620 621 case ISD::ANY_EXTEND_VECTOR_INREG: 622 case ISD::SIGN_EXTEND_VECTOR_INREG: 623 case ISD::ZERO_EXTEND_VECTOR_INREG: 624 SplitVecRes_ExtVecInRegOp(N, Lo, Hi); 625 break; 626 627 case ISD::BITREVERSE: 628 case ISD::BSWAP: 629 case ISD::CONVERT_RNDSAT: 630 case ISD::CTLZ: 631 case ISD::CTTZ: 632 case ISD::CTLZ_ZERO_UNDEF: 633 case ISD::CTTZ_ZERO_UNDEF: 634 case ISD::CTPOP: 635 case ISD::FABS: 636 case ISD::FCEIL: 637 case ISD::FCOS: 638 case ISD::FEXP: 639 case ISD::FEXP2: 640 case ISD::FFLOOR: 641 case ISD::FLOG: 642 case ISD::FLOG10: 643 case ISD::FLOG2: 644 case ISD::FNEARBYINT: 645 case ISD::FNEG: 646 case ISD::FP_EXTEND: 647 case ISD::FP_ROUND: 648 case ISD::FP_TO_SINT: 649 case ISD::FP_TO_UINT: 650 case ISD::FRINT: 651 case ISD::FROUND: 652 case ISD::FSIN: 653 case ISD::FSQRT: 654 case ISD::FTRUNC: 655 case ISD::SINT_TO_FP: 656 case ISD::TRUNCATE: 657 case ISD::UINT_TO_FP: 658 SplitVecRes_UnaryOp(N, Lo, Hi); 659 break; 660 661 case ISD::ANY_EXTEND: 662 case ISD::SIGN_EXTEND: 663 case ISD::ZERO_EXTEND: 664 SplitVecRes_ExtendOp(N, Lo, Hi); 665 break; 666 667 case ISD::ADD: 668 case ISD::SUB: 669 case ISD::MUL: 670 case ISD::MULHS: 671 case ISD::MULHU: 672 case ISD::FADD: 673 case ISD::FSUB: 674 case ISD::FMUL: 675 case ISD::FMINNUM: 676 case ISD::FMAXNUM: 677 case ISD::FMINNAN: 678 case ISD::FMAXNAN: 679 case ISD::SDIV: 680 case ISD::UDIV: 681 case ISD::FDIV: 682 case ISD::FPOW: 683 case ISD::AND: 684 case ISD::OR: 685 case ISD::XOR: 686 case ISD::SHL: 687 case ISD::SRA: 688 case ISD::SRL: 689 case ISD::UREM: 690 case ISD::SREM: 691 case ISD::FREM: 692 case ISD::SMIN: 693 case ISD::SMAX: 694 case ISD::UMIN: 695 case ISD::UMAX: 696 SplitVecRes_BinOp(N, Lo, Hi); 697 break; 698 case ISD::FMA: 699 SplitVecRes_TernaryOp(N, Lo, Hi); 700 break; 701 } 702 703 // If Lo/Hi is null, the sub-method took care of registering results etc. 704 if (Lo.getNode()) 705 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 706 } 707 708 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 709 SDValue &Hi) { 710 SDValue LHSLo, LHSHi; 711 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 712 SDValue RHSLo, RHSHi; 713 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 714 SDLoc dl(N); 715 716 const SDNodeFlags *Flags = N->getFlags(); 717 unsigned Opcode = N->getOpcode(); 718 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags); 719 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags); 720 } 721 722 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, 723 SDValue &Hi) { 724 SDValue Op0Lo, Op0Hi; 725 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 726 SDValue Op1Lo, Op1Hi; 727 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 728 SDValue Op2Lo, Op2Hi; 729 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi); 730 SDLoc dl(N); 731 732 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), 733 Op0Lo, Op1Lo, Op2Lo); 734 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), 735 Op0Hi, Op1Hi, Op2Hi); 736 } 737 738 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, 739 SDValue &Hi) { 740 // We know the result is a vector. The input may be either a vector or a 741 // scalar value. 742 EVT LoVT, HiVT; 743 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 744 SDLoc dl(N); 745 746 SDValue InOp = N->getOperand(0); 747 EVT InVT = InOp.getValueType(); 748 749 // Handle some special cases efficiently. 750 switch (getTypeAction(InVT)) { 751 case TargetLowering::TypeLegal: 752 case TargetLowering::TypePromoteInteger: 753 case TargetLowering::TypePromoteFloat: 754 case TargetLowering::TypeSoftenFloat: 755 case TargetLowering::TypeScalarizeVector: 756 case TargetLowering::TypeWidenVector: 757 break; 758 case TargetLowering::TypeExpandInteger: 759 case TargetLowering::TypeExpandFloat: 760 // A scalar to vector conversion, where the scalar needs expansion. 761 // If the vector is being split in two then we can just convert the 762 // expanded pieces. 763 if (LoVT == HiVT) { 764 GetExpandedOp(InOp, Lo, Hi); 765 if (DAG.getDataLayout().isBigEndian()) 766 std::swap(Lo, Hi); 767 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 768 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 769 return; 770 } 771 break; 772 case TargetLowering::TypeSplitVector: 773 // If the input is a vector that needs to be split, convert each split 774 // piece of the input now. 775 GetSplitVector(InOp, Lo, Hi); 776 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 777 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 778 return; 779 } 780 781 // In the general case, convert the input to an integer and split it by hand. 782 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 783 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 784 if (DAG.getDataLayout().isBigEndian()) 785 std::swap(LoIntVT, HiIntVT); 786 787 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 788 789 if (DAG.getDataLayout().isBigEndian()) 790 std::swap(Lo, Hi); 791 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 792 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 793 } 794 795 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 796 SDValue &Hi) { 797 EVT LoVT, HiVT; 798 SDLoc dl(N); 799 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 800 unsigned LoNumElts = LoVT.getVectorNumElements(); 801 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 802 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, LoOps); 803 804 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 805 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, HiOps); 806 } 807 808 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 809 SDValue &Hi) { 810 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 811 SDLoc dl(N); 812 unsigned NumSubvectors = N->getNumOperands() / 2; 813 if (NumSubvectors == 1) { 814 Lo = N->getOperand(0); 815 Hi = N->getOperand(1); 816 return; 817 } 818 819 EVT LoVT, HiVT; 820 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 821 822 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 823 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps); 824 825 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 826 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps); 827 } 828 829 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 830 SDValue &Hi) { 831 SDValue Vec = N->getOperand(0); 832 SDValue Idx = N->getOperand(1); 833 SDLoc dl(N); 834 835 EVT LoVT, HiVT; 836 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 837 838 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 839 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 840 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, 841 DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl, 842 TLI.getVectorIdxTy(DAG.getDataLayout()))); 843 } 844 845 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, 846 SDValue &Hi) { 847 SDValue Vec = N->getOperand(0); 848 SDValue SubVec = N->getOperand(1); 849 SDValue Idx = N->getOperand(2); 850 SDLoc dl(N); 851 GetSplitVector(Vec, Lo, Hi); 852 853 EVT VecVT = Vec.getValueType(); 854 EVT VecElemVT = VecVT.getVectorElementType(); 855 unsigned VecElems = VecVT.getVectorNumElements(); 856 unsigned SubElems = SubVec.getValueType().getVectorNumElements(); 857 858 // If we know the index is 0, and we know the subvector doesn't cross the 859 // boundary between the halves, we can avoid spilling the vector, and insert 860 // into the lower half of the split vector directly. 861 // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever 862 // the index is constant and there is no boundary crossing. But those cases 863 // don't seem to get hit in practice. 864 if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) { 865 unsigned IdxVal = ConstIdx->getZExtValue(); 866 if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) { 867 EVT LoVT, HiVT; 868 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 869 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx); 870 return; 871 } 872 } 873 874 // Spill the vector to the stack. 875 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 876 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 877 MachinePointerInfo(), false, false, 0); 878 879 // Store the new subvector into the specified index. 880 SDValue SubVecPtr = GetVectorElementPointer(StackPtr, VecElemVT, Idx); 881 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 882 unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType); 883 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo(), 884 false, false, 0); 885 886 // Load the Lo part from the stack slot. 887 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 888 false, false, false, 0); 889 890 // Increment the pointer to the other part. 891 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 892 StackPtr = 893 DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 894 DAG.getConstant(IncrementSize, dl, StackPtr.getValueType())); 895 896 // Load the Hi part from the stack slot. 897 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 898 false, false, false, MinAlign(Alignment, IncrementSize)); 899 } 900 901 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 902 SDValue &Hi) { 903 SDLoc dl(N); 904 GetSplitVector(N->getOperand(0), Lo, Hi); 905 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 906 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 907 } 908 909 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo, 910 SDValue &Hi) { 911 SDValue LHSLo, LHSHi; 912 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 913 SDLoc DL(N); 914 915 SDValue RHSLo, RHSHi; 916 SDValue RHS = N->getOperand(1); 917 EVT RHSVT = RHS.getValueType(); 918 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector) 919 GetSplitVector(RHS, RHSLo, RHSHi); 920 else 921 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS)); 922 923 924 Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo); 925 Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi); 926 } 927 928 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 929 SDValue &Hi) { 930 SDValue LHSLo, LHSHi; 931 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 932 SDLoc dl(N); 933 934 EVT LoVT, HiVT; 935 std::tie(LoVT, HiVT) = 936 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT()); 937 938 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 939 DAG.getValueType(LoVT)); 940 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 941 DAG.getValueType(HiVT)); 942 } 943 944 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, 945 SDValue &Hi) { 946 unsigned Opcode = N->getOpcode(); 947 SDValue N0 = N->getOperand(0); 948 949 SDLoc dl(N); 950 SDValue InLo, InHi; 951 GetSplitVector(N0, InLo, InHi); 952 EVT InLoVT = InLo.getValueType(); 953 unsigned InNumElements = InLoVT.getVectorNumElements(); 954 955 EVT OutLoVT, OutHiVT; 956 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 957 unsigned OutNumElements = OutLoVT.getVectorNumElements(); 958 assert((2 * OutNumElements) <= InNumElements && 959 "Illegal extend vector in reg split"); 960 961 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the 962 // input vector (i.e. we only use InLo): 963 // OutLo will extend the first OutNumElements from InLo. 964 // OutHi will extend the next OutNumElements from InLo. 965 966 // Shuffle the elements from InLo for OutHi into the bottom elements to 967 // create a 'fake' InHi. 968 SmallVector<int, 8> SplitHi(InNumElements, -1); 969 for (unsigned i = 0; i != OutNumElements; ++i) 970 SplitHi[i] = i + OutNumElements; 971 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi); 972 973 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo); 974 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi); 975 } 976 977 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 978 SDValue &Hi) { 979 SDValue Vec = N->getOperand(0); 980 SDValue Elt = N->getOperand(1); 981 SDValue Idx = N->getOperand(2); 982 SDLoc dl(N); 983 GetSplitVector(Vec, Lo, Hi); 984 985 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 986 unsigned IdxVal = CIdx->getZExtValue(); 987 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 988 if (IdxVal < LoNumElts) 989 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 990 Lo.getValueType(), Lo, Elt, Idx); 991 else 992 Hi = 993 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 994 DAG.getConstant(IdxVal - LoNumElts, dl, 995 TLI.getVectorIdxTy(DAG.getDataLayout()))); 996 return; 997 } 998 999 // See if the target wants to custom expand this node. 1000 if (CustomLowerNode(N, N->getValueType(0), true)) 1001 return; 1002 1003 // Spill the vector to the stack. 1004 EVT VecVT = Vec.getValueType(); 1005 EVT EltVT = VecVT.getVectorElementType(); 1006 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1007 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1008 MachinePointerInfo(), false, false, 0); 1009 1010 // Store the new element. This may be larger than the vector element type, 1011 // so use a truncating store. 1012 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1013 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 1014 unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType); 1015 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT, 1016 false, false, 0); 1017 1018 // Load the Lo part from the stack slot. 1019 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 1020 false, false, false, 0); 1021 1022 // Increment the pointer to the other part. 1023 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 1024 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 1025 DAG.getConstant(IncrementSize, dl, 1026 StackPtr.getValueType())); 1027 1028 // Load the Hi part from the stack slot. 1029 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 1030 false, false, false, MinAlign(Alignment, IncrementSize)); 1031 } 1032 1033 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 1034 SDValue &Hi) { 1035 EVT LoVT, HiVT; 1036 SDLoc dl(N); 1037 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1038 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 1039 Hi = DAG.getUNDEF(HiVT); 1040 } 1041 1042 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 1043 SDValue &Hi) { 1044 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 1045 EVT LoVT, HiVT; 1046 SDLoc dl(LD); 1047 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0)); 1048 1049 ISD::LoadExtType ExtType = LD->getExtensionType(); 1050 SDValue Ch = LD->getChain(); 1051 SDValue Ptr = LD->getBasePtr(); 1052 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 1053 EVT MemoryVT = LD->getMemoryVT(); 1054 unsigned Alignment = LD->getOriginalAlignment(); 1055 bool isVolatile = LD->isVolatile(); 1056 bool isNonTemporal = LD->isNonTemporal(); 1057 bool isInvariant = LD->isInvariant(); 1058 AAMDNodes AAInfo = LD->getAAInfo(); 1059 1060 EVT LoMemVT, HiMemVT; 1061 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1062 1063 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset, 1064 LD->getPointerInfo(), LoMemVT, isVolatile, isNonTemporal, 1065 isInvariant, Alignment, AAInfo); 1066 1067 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1068 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1069 DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); 1070 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, 1071 LD->getPointerInfo().getWithOffset(IncrementSize), 1072 HiMemVT, isVolatile, isNonTemporal, isInvariant, Alignment, 1073 AAInfo); 1074 1075 // Build a factor node to remember that this load is independent of the 1076 // other one. 1077 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1078 Hi.getValue(1)); 1079 1080 // Legalize the chain result - switch anything that used the old chain to 1081 // use the new one. 1082 ReplaceValueWith(SDValue(LD, 1), Ch); 1083 } 1084 1085 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, 1086 SDValue &Lo, SDValue &Hi) { 1087 EVT LoVT, HiVT; 1088 SDLoc dl(MLD); 1089 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0)); 1090 1091 SDValue Ch = MLD->getChain(); 1092 SDValue Ptr = MLD->getBasePtr(); 1093 SDValue Mask = MLD->getMask(); 1094 SDValue Src0 = MLD->getSrc0(); 1095 unsigned Alignment = MLD->getOriginalAlignment(); 1096 ISD::LoadExtType ExtType = MLD->getExtensionType(); 1097 1098 // if Alignment is equal to the vector size, 1099 // take the half of it for the second part 1100 unsigned SecondHalfAlignment = 1101 (Alignment == MLD->getValueType(0).getSizeInBits()/8) ? 1102 Alignment/2 : Alignment; 1103 1104 // Split Mask operand 1105 SDValue MaskLo, MaskHi; 1106 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 1107 GetSplitVector(Mask, MaskLo, MaskHi); 1108 else 1109 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 1110 1111 EVT MemoryVT = MLD->getMemoryVT(); 1112 EVT LoMemVT, HiMemVT; 1113 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1114 1115 SDValue Src0Lo, Src0Hi; 1116 if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector) 1117 GetSplitVector(Src0, Src0Lo, Src0Hi); 1118 else 1119 std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl); 1120 1121 MachineMemOperand *MMO = DAG.getMachineFunction(). 1122 getMachineMemOperand(MLD->getPointerInfo(), 1123 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(), 1124 Alignment, MLD->getAAInfo(), MLD->getRanges()); 1125 1126 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, Src0Lo, LoMemVT, MMO, 1127 ExtType); 1128 1129 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1130 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1131 DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); 1132 1133 MMO = DAG.getMachineFunction(). 1134 getMachineMemOperand(MLD->getPointerInfo(), 1135 MachineMemOperand::MOLoad, HiMemVT.getStoreSize(), 1136 SecondHalfAlignment, MLD->getAAInfo(), MLD->getRanges()); 1137 1138 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, Src0Hi, HiMemVT, MMO, 1139 ExtType); 1140 1141 1142 // Build a factor node to remember that this load is independent of the 1143 // other one. 1144 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1145 Hi.getValue(1)); 1146 1147 // Legalize the chain result - switch anything that used the old chain to 1148 // use the new one. 1149 ReplaceValueWith(SDValue(MLD, 1), Ch); 1150 1151 } 1152 1153 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT, 1154 SDValue &Lo, SDValue &Hi) { 1155 EVT LoVT, HiVT; 1156 SDLoc dl(MGT); 1157 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0)); 1158 1159 SDValue Ch = MGT->getChain(); 1160 SDValue Ptr = MGT->getBasePtr(); 1161 SDValue Mask = MGT->getMask(); 1162 SDValue Src0 = MGT->getValue(); 1163 SDValue Index = MGT->getIndex(); 1164 unsigned Alignment = MGT->getOriginalAlignment(); 1165 1166 // Split Mask operand 1167 SDValue MaskLo, MaskHi; 1168 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 1169 GetSplitVector(Mask, MaskLo, MaskHi); 1170 else 1171 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 1172 1173 EVT MemoryVT = MGT->getMemoryVT(); 1174 EVT LoMemVT, HiMemVT; 1175 // Split MemoryVT 1176 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1177 1178 SDValue Src0Lo, Src0Hi; 1179 if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector) 1180 GetSplitVector(Src0, Src0Lo, Src0Hi); 1181 else 1182 std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl); 1183 1184 SDValue IndexHi, IndexLo; 1185 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector) 1186 GetSplitVector(Index, IndexLo, IndexHi); 1187 else 1188 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl); 1189 1190 MachineMemOperand *MMO = DAG.getMachineFunction(). 1191 getMachineMemOperand(MGT->getPointerInfo(), 1192 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(), 1193 Alignment, MGT->getAAInfo(), MGT->getRanges()); 1194 1195 SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo}; 1196 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo, 1197 MMO); 1198 1199 SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi}; 1200 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi, 1201 MMO); 1202 1203 // Build a factor node to remember that this load is independent of the 1204 // other one. 1205 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1206 Hi.getValue(1)); 1207 1208 // Legalize the chain result - switch anything that used the old chain to 1209 // use the new one. 1210 ReplaceValueWith(SDValue(MGT, 1), Ch); 1211 } 1212 1213 1214 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 1215 assert(N->getValueType(0).isVector() && 1216 N->getOperand(0).getValueType().isVector() && 1217 "Operand types must be vectors"); 1218 1219 EVT LoVT, HiVT; 1220 SDLoc DL(N); 1221 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1222 1223 // Split the input. 1224 SDValue LL, LH, RL, RH; 1225 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0); 1226 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1); 1227 1228 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 1229 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 1230 } 1231 1232 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 1233 SDValue &Hi) { 1234 // Get the dest types - they may not match the input types, e.g. int_to_fp. 1235 EVT LoVT, HiVT; 1236 SDLoc dl(N); 1237 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1238 1239 // If the input also splits, handle it directly for a compile time speedup. 1240 // Otherwise split it by hand. 1241 EVT InVT = N->getOperand(0).getValueType(); 1242 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) 1243 GetSplitVector(N->getOperand(0), Lo, Hi); 1244 else 1245 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); 1246 1247 if (N->getOpcode() == ISD::FP_ROUND) { 1248 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1)); 1249 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1)); 1250 } else if (N->getOpcode() == ISD::CONVERT_RNDSAT) { 1251 SDValue DTyOpLo = DAG.getValueType(LoVT); 1252 SDValue DTyOpHi = DAG.getValueType(HiVT); 1253 SDValue STyOpLo = DAG.getValueType(Lo.getValueType()); 1254 SDValue STyOpHi = DAG.getValueType(Hi.getValueType()); 1255 SDValue RndOp = N->getOperand(3); 1256 SDValue SatOp = N->getOperand(4); 1257 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1258 Lo = DAG.getConvertRndSat(LoVT, dl, Lo, DTyOpLo, STyOpLo, RndOp, SatOp, 1259 CvtCode); 1260 Hi = DAG.getConvertRndSat(HiVT, dl, Hi, DTyOpHi, STyOpHi, RndOp, SatOp, 1261 CvtCode); 1262 } else { 1263 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 1264 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 1265 } 1266 } 1267 1268 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, 1269 SDValue &Hi) { 1270 SDLoc dl(N); 1271 EVT SrcVT = N->getOperand(0).getValueType(); 1272 EVT DestVT = N->getValueType(0); 1273 EVT LoVT, HiVT; 1274 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT); 1275 1276 // We can do better than a generic split operation if the extend is doing 1277 // more than just doubling the width of the elements and the following are 1278 // true: 1279 // - The number of vector elements is even, 1280 // - the source type is legal, 1281 // - the type of a split source is illegal, 1282 // - the type of an extended (by doubling element size) source is legal, and 1283 // - the type of that extended source when split is legal. 1284 // 1285 // This won't necessarily completely legalize the operation, but it will 1286 // more effectively move in the right direction and prevent falling down 1287 // to scalarization in many cases due to the input vector being split too 1288 // far. 1289 unsigned NumElements = SrcVT.getVectorNumElements(); 1290 if ((NumElements & 1) == 0 && 1291 SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) { 1292 LLVMContext &Ctx = *DAG.getContext(); 1293 EVT NewSrcVT = EVT::getVectorVT( 1294 Ctx, EVT::getIntegerVT( 1295 Ctx, SrcVT.getVectorElementType().getSizeInBits() * 2), 1296 NumElements); 1297 EVT SplitSrcVT = 1298 EVT::getVectorVT(Ctx, SrcVT.getVectorElementType(), NumElements / 2); 1299 EVT SplitLoVT, SplitHiVT; 1300 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT); 1301 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) && 1302 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) { 1303 DEBUG(dbgs() << "Split vector extend via incremental extend:"; 1304 N->dump(&DAG); dbgs() << "\n"); 1305 // Extend the source vector by one step. 1306 SDValue NewSrc = 1307 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0)); 1308 // Get the low and high halves of the new, extended one step, vector. 1309 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl); 1310 // Extend those vector halves the rest of the way. 1311 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 1312 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 1313 return; 1314 } 1315 } 1316 // Fall back to the generic unary operator splitting otherwise. 1317 SplitVecRes_UnaryOp(N, Lo, Hi); 1318 } 1319 1320 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 1321 SDValue &Lo, SDValue &Hi) { 1322 // The low and high parts of the original input give four input vectors. 1323 SDValue Inputs[4]; 1324 SDLoc dl(N); 1325 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 1326 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 1327 EVT NewVT = Inputs[0].getValueType(); 1328 unsigned NewElts = NewVT.getVectorNumElements(); 1329 1330 // If Lo or Hi uses elements from at most two of the four input vectors, then 1331 // express it as a vector shuffle of those two inputs. Otherwise extract the 1332 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 1333 SmallVector<int, 16> Ops; 1334 for (unsigned High = 0; High < 2; ++High) { 1335 SDValue &Output = High ? Hi : Lo; 1336 1337 // Build a shuffle mask for the output, discovering on the fly which 1338 // input vectors to use as shuffle operands (recorded in InputUsed). 1339 // If building a suitable shuffle vector proves too hard, then bail 1340 // out with useBuildVector set. 1341 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 1342 unsigned FirstMaskIdx = High * NewElts; 1343 bool useBuildVector = false; 1344 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 1345 // The mask element. This indexes into the input. 1346 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 1347 1348 // The input vector this mask element indexes into. 1349 unsigned Input = (unsigned)Idx / NewElts; 1350 1351 if (Input >= array_lengthof(Inputs)) { 1352 // The mask element does not index into any input vector. 1353 Ops.push_back(-1); 1354 continue; 1355 } 1356 1357 // Turn the index into an offset from the start of the input vector. 1358 Idx -= Input * NewElts; 1359 1360 // Find or create a shuffle vector operand to hold this input. 1361 unsigned OpNo; 1362 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 1363 if (InputUsed[OpNo] == Input) { 1364 // This input vector is already an operand. 1365 break; 1366 } else if (InputUsed[OpNo] == -1U) { 1367 // Create a new operand for this input vector. 1368 InputUsed[OpNo] = Input; 1369 break; 1370 } 1371 } 1372 1373 if (OpNo >= array_lengthof(InputUsed)) { 1374 // More than two input vectors used! Give up on trying to create a 1375 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 1376 useBuildVector = true; 1377 break; 1378 } 1379 1380 // Add the mask index for the new shuffle vector. 1381 Ops.push_back(Idx + OpNo * NewElts); 1382 } 1383 1384 if (useBuildVector) { 1385 EVT EltVT = NewVT.getVectorElementType(); 1386 SmallVector<SDValue, 16> SVOps; 1387 1388 // Extract the input elements by hand. 1389 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 1390 // The mask element. This indexes into the input. 1391 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 1392 1393 // The input vector this mask element indexes into. 1394 unsigned Input = (unsigned)Idx / NewElts; 1395 1396 if (Input >= array_lengthof(Inputs)) { 1397 // The mask element is "undef" or indexes off the end of the input. 1398 SVOps.push_back(DAG.getUNDEF(EltVT)); 1399 continue; 1400 } 1401 1402 // Turn the index into an offset from the start of the input vector. 1403 Idx -= Input * NewElts; 1404 1405 // Extract the vector element by hand. 1406 SVOps.push_back(DAG.getNode( 1407 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input], 1408 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())))); 1409 } 1410 1411 // Construct the Lo/Hi output using a BUILD_VECTOR. 1412 Output = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, SVOps); 1413 } else if (InputUsed[0] == -1U) { 1414 // No input vectors were used! The result is undefined. 1415 Output = DAG.getUNDEF(NewVT); 1416 } else { 1417 SDValue Op0 = Inputs[InputUsed[0]]; 1418 // If only one input was used, use an undefined vector for the other. 1419 SDValue Op1 = InputUsed[1] == -1U ? 1420 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 1421 // At least one input vector was used. Create a new shuffle vector. 1422 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops); 1423 } 1424 1425 Ops.clear(); 1426 } 1427 } 1428 1429 1430 //===----------------------------------------------------------------------===// 1431 // Operand Vector Splitting 1432 //===----------------------------------------------------------------------===// 1433 1434 /// This method is called when the specified operand of the specified node is 1435 /// found to need vector splitting. At this point, all of the result types of 1436 /// the node are known to be legal, but other operands of the node may need 1437 /// legalization as well as the specified one. 1438 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 1439 DEBUG(dbgs() << "Split node operand: "; 1440 N->dump(&DAG); 1441 dbgs() << "\n"); 1442 SDValue Res = SDValue(); 1443 1444 // See if the target wants to custom split this node. 1445 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 1446 return false; 1447 1448 if (!Res.getNode()) { 1449 switch (N->getOpcode()) { 1450 default: 1451 #ifndef NDEBUG 1452 dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 1453 N->dump(&DAG); 1454 dbgs() << "\n"; 1455 #endif 1456 report_fatal_error("Do not know how to split this operator's " 1457 "operand!\n"); 1458 1459 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; 1460 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; 1461 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 1462 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 1463 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; 1464 case ISD::TRUNCATE: 1465 Res = SplitVecOp_TruncateHelper(N); 1466 break; 1467 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; 1468 case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break; 1469 case ISD::STORE: 1470 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 1471 break; 1472 case ISD::MSTORE: 1473 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo); 1474 break; 1475 case ISD::MSCATTER: 1476 Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo); 1477 break; 1478 case ISD::MGATHER: 1479 Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo); 1480 break; 1481 case ISD::VSELECT: 1482 Res = SplitVecOp_VSELECT(N, OpNo); 1483 break; 1484 case ISD::FP_TO_SINT: 1485 case ISD::FP_TO_UINT: 1486 if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0))) 1487 Res = SplitVecOp_TruncateHelper(N); 1488 else 1489 Res = SplitVecOp_UnaryOp(N); 1490 break; 1491 case ISD::SINT_TO_FP: 1492 case ISD::UINT_TO_FP: 1493 if (N->getValueType(0).bitsLT(N->getOperand(0)->getValueType(0))) 1494 Res = SplitVecOp_TruncateHelper(N); 1495 else 1496 Res = SplitVecOp_UnaryOp(N); 1497 break; 1498 case ISD::CTTZ: 1499 case ISD::CTLZ: 1500 case ISD::CTPOP: 1501 case ISD::FP_EXTEND: 1502 case ISD::SIGN_EXTEND: 1503 case ISD::ZERO_EXTEND: 1504 case ISD::ANY_EXTEND: 1505 case ISD::FTRUNC: 1506 Res = SplitVecOp_UnaryOp(N); 1507 break; 1508 } 1509 } 1510 1511 // If the result is null, the sub-method took care of registering results etc. 1512 if (!Res.getNode()) return false; 1513 1514 // If the result is N, the sub-method updated N in place. Tell the legalizer 1515 // core about this. 1516 if (Res.getNode() == N) 1517 return true; 1518 1519 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1520 "Invalid operand expansion"); 1521 1522 ReplaceValueWith(SDValue(N, 0), Res); 1523 return false; 1524 } 1525 1526 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { 1527 // The only possibility for an illegal operand is the mask, since result type 1528 // legalization would have handled this node already otherwise. 1529 assert(OpNo == 0 && "Illegal operand must be mask"); 1530 1531 SDValue Mask = N->getOperand(0); 1532 SDValue Src0 = N->getOperand(1); 1533 SDValue Src1 = N->getOperand(2); 1534 EVT Src0VT = Src0.getValueType(); 1535 SDLoc DL(N); 1536 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?"); 1537 1538 SDValue Lo, Hi; 1539 GetSplitVector(N->getOperand(0), Lo, Hi); 1540 assert(Lo.getValueType() == Hi.getValueType() && 1541 "Lo and Hi have differing types"); 1542 1543 EVT LoOpVT, HiOpVT; 1544 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT); 1545 assert(LoOpVT == HiOpVT && "Asymmetric vector split?"); 1546 1547 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask; 1548 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL); 1549 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL); 1550 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL); 1551 1552 SDValue LoSelect = 1553 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1); 1554 SDValue HiSelect = 1555 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1); 1556 1557 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect); 1558 } 1559 1560 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 1561 // The result has a legal vector type, but the input needs splitting. 1562 EVT ResVT = N->getValueType(0); 1563 SDValue Lo, Hi; 1564 SDLoc dl(N); 1565 GetSplitVector(N->getOperand(0), Lo, Hi); 1566 EVT InVT = Lo.getValueType(); 1567 1568 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1569 InVT.getVectorNumElements()); 1570 1571 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 1572 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 1573 1574 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 1575 } 1576 1577 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { 1578 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will 1579 // end up being split all the way down to individual components. Convert the 1580 // split pieces into integers and reassemble. 1581 SDValue Lo, Hi; 1582 GetSplitVector(N->getOperand(0), Lo, Hi); 1583 Lo = BitConvertToInteger(Lo); 1584 Hi = BitConvertToInteger(Hi); 1585 1586 if (DAG.getDataLayout().isBigEndian()) 1587 std::swap(Lo, Hi); 1588 1589 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), 1590 JoinIntegers(Lo, Hi)); 1591 } 1592 1593 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1594 // We know that the extracted result type is legal. 1595 EVT SubVT = N->getValueType(0); 1596 SDValue Idx = N->getOperand(1); 1597 SDLoc dl(N); 1598 SDValue Lo, Hi; 1599 GetSplitVector(N->getOperand(0), Lo, Hi); 1600 1601 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1602 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1603 1604 if (IdxVal < LoElts) { 1605 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1606 "Extracted subvector crosses vector split!"); 1607 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1608 } else { 1609 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1610 DAG.getConstant(IdxVal - LoElts, dl, 1611 Idx.getValueType())); 1612 } 1613 } 1614 1615 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1616 SDValue Vec = N->getOperand(0); 1617 SDValue Idx = N->getOperand(1); 1618 EVT VecVT = Vec.getValueType(); 1619 1620 if (isa<ConstantSDNode>(Idx)) { 1621 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1622 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1623 1624 SDValue Lo, Hi; 1625 GetSplitVector(Vec, Lo, Hi); 1626 1627 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1628 1629 if (IdxVal < LoElts) 1630 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0); 1631 return SDValue(DAG.UpdateNodeOperands(N, Hi, 1632 DAG.getConstant(IdxVal - LoElts, SDLoc(N), 1633 Idx.getValueType())), 0); 1634 } 1635 1636 // See if the target wants to custom expand this node. 1637 if (CustomLowerNode(N, N->getValueType(0), true)) 1638 return SDValue(); 1639 1640 // Make the vector elements byte-addressable if they aren't already. 1641 SDLoc dl(N); 1642 EVT EltVT = VecVT.getVectorElementType(); 1643 if (EltVT.getSizeInBits() < 8) { 1644 SmallVector<SDValue, 4> ElementOps; 1645 for (unsigned i = 0; i < VecVT.getVectorNumElements(); ++i) { 1646 ElementOps.push_back(DAG.getAnyExtOrTrunc( 1647 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vec, 1648 DAG.getConstant(i, dl, MVT::i8)), 1649 dl, MVT::i8)); 1650 } 1651 1652 EltVT = MVT::i8; 1653 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 1654 VecVT.getVectorNumElements()); 1655 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, VecVT, ElementOps); 1656 } 1657 1658 // Store the vector to the stack. 1659 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1660 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1661 MachinePointerInfo(), false, false, 0); 1662 1663 // Load back the required element. 1664 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1665 return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 1666 MachinePointerInfo(), EltVT, false, false, false, 0); 1667 } 1668 1669 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT, 1670 unsigned OpNo) { 1671 EVT LoVT, HiVT; 1672 SDLoc dl(MGT); 1673 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0)); 1674 1675 SDValue Ch = MGT->getChain(); 1676 SDValue Ptr = MGT->getBasePtr(); 1677 SDValue Index = MGT->getIndex(); 1678 SDValue Mask = MGT->getMask(); 1679 SDValue Src0 = MGT->getValue(); 1680 unsigned Alignment = MGT->getOriginalAlignment(); 1681 1682 SDValue MaskLo, MaskHi; 1683 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 1684 // Split Mask operand 1685 GetSplitVector(Mask, MaskLo, MaskHi); 1686 else 1687 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 1688 1689 EVT MemoryVT = MGT->getMemoryVT(); 1690 EVT LoMemVT, HiMemVT; 1691 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1692 1693 SDValue Src0Lo, Src0Hi; 1694 if (getTypeAction(Src0.getValueType()) == TargetLowering::TypeSplitVector) 1695 GetSplitVector(Src0, Src0Lo, Src0Hi); 1696 else 1697 std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, dl); 1698 1699 SDValue IndexHi, IndexLo; 1700 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector) 1701 GetSplitVector(Index, IndexLo, IndexHi); 1702 else 1703 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl); 1704 1705 MachineMemOperand *MMO = DAG.getMachineFunction(). 1706 getMachineMemOperand(MGT->getPointerInfo(), 1707 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(), 1708 Alignment, MGT->getAAInfo(), MGT->getRanges()); 1709 1710 SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo}; 1711 SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, 1712 OpsLo, MMO); 1713 1714 MMO = DAG.getMachineFunction(). 1715 getMachineMemOperand(MGT->getPointerInfo(), 1716 MachineMemOperand::MOLoad, HiMemVT.getStoreSize(), 1717 Alignment, MGT->getAAInfo(), 1718 MGT->getRanges()); 1719 1720 SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi}; 1721 SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, 1722 OpsHi, MMO); 1723 1724 // Build a factor node to remember that this load is independent of the 1725 // other one. 1726 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1727 Hi.getValue(1)); 1728 1729 // Legalize the chain result - switch anything that used the old chain to 1730 // use the new one. 1731 ReplaceValueWith(SDValue(MGT, 1), Ch); 1732 1733 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo, 1734 Hi); 1735 ReplaceValueWith(SDValue(MGT, 0), Res); 1736 return SDValue(); 1737 } 1738 1739 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N, 1740 unsigned OpNo) { 1741 SDValue Ch = N->getChain(); 1742 SDValue Ptr = N->getBasePtr(); 1743 SDValue Mask = N->getMask(); 1744 SDValue Data = N->getValue(); 1745 EVT MemoryVT = N->getMemoryVT(); 1746 unsigned Alignment = N->getOriginalAlignment(); 1747 SDLoc DL(N); 1748 1749 EVT LoMemVT, HiMemVT; 1750 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1751 1752 SDValue DataLo, DataHi; 1753 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector) 1754 // Split Data operand 1755 GetSplitVector(Data, DataLo, DataHi); 1756 else 1757 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL); 1758 1759 SDValue MaskLo, MaskHi; 1760 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 1761 // Split Mask operand 1762 GetSplitVector(Mask, MaskLo, MaskHi); 1763 else 1764 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL); 1765 1766 MaskLo = PromoteTargetBoolean(MaskLo, DataLo.getValueType()); 1767 MaskHi = PromoteTargetBoolean(MaskHi, DataHi.getValueType()); 1768 1769 // if Alignment is equal to the vector size, 1770 // take the half of it for the second part 1771 unsigned SecondHalfAlignment = 1772 (Alignment == Data->getValueType(0).getSizeInBits()/8) ? 1773 Alignment/2 : Alignment; 1774 1775 SDValue Lo, Hi; 1776 MachineMemOperand *MMO = DAG.getMachineFunction(). 1777 getMachineMemOperand(N->getPointerInfo(), 1778 MachineMemOperand::MOStore, LoMemVT.getStoreSize(), 1779 Alignment, N->getAAInfo(), N->getRanges()); 1780 1781 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO, 1782 N->isTruncatingStore()); 1783 1784 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1785 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 1786 DAG.getConstant(IncrementSize, DL, Ptr.getValueType())); 1787 1788 MMO = DAG.getMachineFunction(). 1789 getMachineMemOperand(N->getPointerInfo(), 1790 MachineMemOperand::MOStore, HiMemVT.getStoreSize(), 1791 SecondHalfAlignment, N->getAAInfo(), N->getRanges()); 1792 1793 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO, 1794 N->isTruncatingStore()); 1795 1796 // Build a factor node to remember that this store is independent of the 1797 // other one. 1798 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 1799 } 1800 1801 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N, 1802 unsigned OpNo) { 1803 SDValue Ch = N->getChain(); 1804 SDValue Ptr = N->getBasePtr(); 1805 SDValue Mask = N->getMask(); 1806 SDValue Index = N->getIndex(); 1807 SDValue Data = N->getValue(); 1808 EVT MemoryVT = N->getMemoryVT(); 1809 unsigned Alignment = N->getOriginalAlignment(); 1810 SDLoc DL(N); 1811 1812 // Split all operands 1813 EVT LoMemVT, HiMemVT; 1814 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1815 1816 SDValue DataLo, DataHi; 1817 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector) 1818 // Split Data operand 1819 GetSplitVector(Data, DataLo, DataHi); 1820 else 1821 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL); 1822 1823 SDValue MaskLo, MaskHi; 1824 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 1825 // Split Mask operand 1826 GetSplitVector(Mask, MaskLo, MaskHi); 1827 else 1828 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL); 1829 1830 SDValue IndexHi, IndexLo; 1831 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector) 1832 GetSplitVector(Index, IndexLo, IndexHi); 1833 else 1834 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL); 1835 1836 SDValue Lo, Hi; 1837 MachineMemOperand *MMO = DAG.getMachineFunction(). 1838 getMachineMemOperand(N->getPointerInfo(), 1839 MachineMemOperand::MOStore, LoMemVT.getStoreSize(), 1840 Alignment, N->getAAInfo(), N->getRanges()); 1841 1842 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo}; 1843 Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(), 1844 DL, OpsLo, MMO); 1845 1846 MMO = DAG.getMachineFunction(). 1847 getMachineMemOperand(N->getPointerInfo(), 1848 MachineMemOperand::MOStore, HiMemVT.getStoreSize(), 1849 Alignment, N->getAAInfo(), N->getRanges()); 1850 1851 SDValue OpsHi[] = {Ch, DataHi, MaskHi, Ptr, IndexHi}; 1852 Hi = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(), 1853 DL, OpsHi, MMO); 1854 1855 // Build a factor node to remember that this store is independent of the 1856 // other one. 1857 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 1858 } 1859 1860 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1861 assert(N->isUnindexed() && "Indexed store of vector?"); 1862 assert(OpNo == 1 && "Can only split the stored value"); 1863 SDLoc DL(N); 1864 1865 bool isTruncating = N->isTruncatingStore(); 1866 SDValue Ch = N->getChain(); 1867 SDValue Ptr = N->getBasePtr(); 1868 EVT MemoryVT = N->getMemoryVT(); 1869 unsigned Alignment = N->getOriginalAlignment(); 1870 bool isVol = N->isVolatile(); 1871 bool isNT = N->isNonTemporal(); 1872 AAMDNodes AAInfo = N->getAAInfo(); 1873 SDValue Lo, Hi; 1874 GetSplitVector(N->getOperand(1), Lo, Hi); 1875 1876 EVT LoMemVT, HiMemVT; 1877 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1878 1879 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1880 1881 if (isTruncating) 1882 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1883 LoMemVT, isVol, isNT, Alignment, AAInfo); 1884 else 1885 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1886 isVol, isNT, Alignment, AAInfo); 1887 1888 // Increment the pointer to the other half. 1889 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 1890 DAG.getConstant(IncrementSize, DL, Ptr.getValueType())); 1891 1892 if (isTruncating) 1893 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, 1894 N->getPointerInfo().getWithOffset(IncrementSize), 1895 HiMemVT, isVol, isNT, Alignment, AAInfo); 1896 else 1897 Hi = DAG.getStore(Ch, DL, Hi, Ptr, 1898 N->getPointerInfo().getWithOffset(IncrementSize), 1899 isVol, isNT, Alignment, AAInfo); 1900 1901 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 1902 } 1903 1904 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { 1905 SDLoc DL(N); 1906 1907 // The input operands all must have the same type, and we know the result 1908 // type is valid. Convert this to a buildvector which extracts all the 1909 // input elements. 1910 // TODO: If the input elements are power-two vectors, we could convert this to 1911 // a new CONCAT_VECTORS node with elements that are half-wide. 1912 SmallVector<SDValue, 32> Elts; 1913 EVT EltVT = N->getValueType(0).getVectorElementType(); 1914 for (const SDValue &Op : N->op_values()) { 1915 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements(); 1916 i != e; ++i) { 1917 Elts.push_back(DAG.getNode( 1918 ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op, 1919 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())))); 1920 } 1921 } 1922 1923 return DAG.getNode(ISD::BUILD_VECTOR, DL, N->getValueType(0), Elts); 1924 } 1925 1926 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) { 1927 // The result type is legal, but the input type is illegal. If splitting 1928 // ends up with the result type of each half still being legal, just 1929 // do that. If, however, that would result in an illegal result type, 1930 // we can try to get more clever with power-two vectors. Specifically, 1931 // split the input type, but also widen the result element size, then 1932 // concatenate the halves and truncate again. For example, consider a target 1933 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit 1934 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do: 1935 // %inlo = v4i32 extract_subvector %in, 0 1936 // %inhi = v4i32 extract_subvector %in, 4 1937 // %lo16 = v4i16 trunc v4i32 %inlo 1938 // %hi16 = v4i16 trunc v4i32 %inhi 1939 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16 1940 // %res = v8i8 trunc v8i16 %in16 1941 // 1942 // Without this transform, the original truncate would end up being 1943 // scalarized, which is pretty much always a last resort. 1944 SDValue InVec = N->getOperand(0); 1945 EVT InVT = InVec->getValueType(0); 1946 EVT OutVT = N->getValueType(0); 1947 unsigned NumElements = OutVT.getVectorNumElements(); 1948 bool IsFloat = OutVT.isFloatingPoint(); 1949 1950 // Widening should have already made sure this is a power-two vector 1951 // if we're trying to split it at all. assert() that's true, just in case. 1952 assert(!(NumElements & 1) && "Splitting vector, but not in half!"); 1953 1954 unsigned InElementSize = InVT.getVectorElementType().getSizeInBits(); 1955 unsigned OutElementSize = OutVT.getVectorElementType().getSizeInBits(); 1956 1957 // If the input elements are only 1/2 the width of the result elements, 1958 // just use the normal splitting. Our trick only work if there's room 1959 // to split more than once. 1960 if (InElementSize <= OutElementSize * 2) 1961 return SplitVecOp_UnaryOp(N); 1962 SDLoc DL(N); 1963 1964 // Extract the halves of the input via extract_subvector. 1965 SDValue InLoVec, InHiVec; 1966 std::tie(InLoVec, InHiVec) = DAG.SplitVector(InVec, DL); 1967 // Truncate them to 1/2 the element size. 1968 EVT HalfElementVT = IsFloat ? 1969 EVT::getFloatingPointVT(InElementSize/2) : 1970 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2); 1971 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, 1972 NumElements/2); 1973 SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec); 1974 SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec); 1975 // Concatenate them to get the full intermediate truncation result. 1976 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements); 1977 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo, 1978 HalfHi); 1979 // Now finish up by truncating all the way down to the original result 1980 // type. This should normally be something that ends up being legal directly, 1981 // but in theory if a target has very wide vectors and an annoyingly 1982 // restricted set of legal types, this split can chain to build things up. 1983 return IsFloat 1984 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec, 1985 DAG.getTargetConstant( 1986 0, DL, TLI.getPointerTy(DAG.getDataLayout()))) 1987 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec); 1988 } 1989 1990 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { 1991 assert(N->getValueType(0).isVector() && 1992 N->getOperand(0).getValueType().isVector() && 1993 "Operand types must be vectors"); 1994 // The result has a legal vector type, but the input needs splitting. 1995 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; 1996 SDLoc DL(N); 1997 GetSplitVector(N->getOperand(0), Lo0, Hi0); 1998 GetSplitVector(N->getOperand(1), Lo1, Hi1); 1999 unsigned PartElements = Lo0.getValueType().getVectorNumElements(); 2000 EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements); 2001 EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements); 2002 2003 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2)); 2004 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2)); 2005 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes); 2006 return PromoteTargetBoolean(Con, N->getValueType(0)); 2007 } 2008 2009 2010 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { 2011 // The result has a legal vector type, but the input needs splitting. 2012 EVT ResVT = N->getValueType(0); 2013 SDValue Lo, Hi; 2014 SDLoc DL(N); 2015 GetSplitVector(N->getOperand(0), Lo, Hi); 2016 EVT InVT = Lo.getValueType(); 2017 2018 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 2019 InVT.getVectorNumElements()); 2020 2021 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); 2022 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); 2023 2024 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); 2025 } 2026 2027 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) { 2028 // The result (and the first input) has a legal vector type, but the second 2029 // input needs splitting. 2030 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements()); 2031 } 2032 2033 2034 //===----------------------------------------------------------------------===// 2035 // Result Vector Widening 2036 //===----------------------------------------------------------------------===// 2037 2038 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 2039 DEBUG(dbgs() << "Widen node result " << ResNo << ": "; 2040 N->dump(&DAG); 2041 dbgs() << "\n"); 2042 2043 // See if the target wants to custom widen this node. 2044 if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 2045 return; 2046 2047 SDValue Res = SDValue(); 2048 switch (N->getOpcode()) { 2049 default: 2050 #ifndef NDEBUG 2051 dbgs() << "WidenVectorResult #" << ResNo << ": "; 2052 N->dump(&DAG); 2053 dbgs() << "\n"; 2054 #endif 2055 llvm_unreachable("Do not know how to widen the result of this operator!"); 2056 2057 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; 2058 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; 2059 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 2060 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 2061 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 2062 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 2063 case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break; 2064 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 2065 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 2066 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 2067 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 2068 case ISD::VSELECT: 2069 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 2070 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 2071 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break; 2072 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 2073 case ISD::VECTOR_SHUFFLE: 2074 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 2075 break; 2076 case ISD::MLOAD: 2077 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N)); 2078 break; 2079 case ISD::MGATHER: 2080 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N)); 2081 break; 2082 2083 case ISD::ADD: 2084 case ISD::AND: 2085 case ISD::MUL: 2086 case ISD::MULHS: 2087 case ISD::MULHU: 2088 case ISD::OR: 2089 case ISD::SUB: 2090 case ISD::XOR: 2091 case ISD::FMINNUM: 2092 case ISD::FMAXNUM: 2093 case ISD::FMINNAN: 2094 case ISD::FMAXNAN: 2095 case ISD::SMIN: 2096 case ISD::SMAX: 2097 case ISD::UMIN: 2098 case ISD::UMAX: 2099 Res = WidenVecRes_Binary(N); 2100 break; 2101 2102 case ISD::FADD: 2103 case ISD::FMUL: 2104 case ISD::FPOW: 2105 case ISD::FSUB: 2106 case ISD::FDIV: 2107 case ISD::FREM: 2108 case ISD::SDIV: 2109 case ISD::UDIV: 2110 case ISD::SREM: 2111 case ISD::UREM: 2112 Res = WidenVecRes_BinaryCanTrap(N); 2113 break; 2114 2115 case ISD::FCOPYSIGN: 2116 Res = WidenVecRes_FCOPYSIGN(N); 2117 break; 2118 2119 case ISD::FPOWI: 2120 Res = WidenVecRes_POWI(N); 2121 break; 2122 2123 case ISD::SHL: 2124 case ISD::SRA: 2125 case ISD::SRL: 2126 Res = WidenVecRes_Shift(N); 2127 break; 2128 2129 case ISD::ANY_EXTEND_VECTOR_INREG: 2130 case ISD::SIGN_EXTEND_VECTOR_INREG: 2131 case ISD::ZERO_EXTEND_VECTOR_INREG: 2132 Res = WidenVecRes_EXTEND_VECTOR_INREG(N); 2133 break; 2134 2135 case ISD::ANY_EXTEND: 2136 case ISD::FP_EXTEND: 2137 case ISD::FP_ROUND: 2138 case ISD::FP_TO_SINT: 2139 case ISD::FP_TO_UINT: 2140 case ISD::SIGN_EXTEND: 2141 case ISD::SINT_TO_FP: 2142 case ISD::TRUNCATE: 2143 case ISD::UINT_TO_FP: 2144 case ISD::ZERO_EXTEND: 2145 Res = WidenVecRes_Convert(N); 2146 break; 2147 2148 case ISD::BITREVERSE: 2149 case ISD::BSWAP: 2150 case ISD::CTLZ: 2151 case ISD::CTPOP: 2152 case ISD::CTTZ: 2153 case ISD::FABS: 2154 case ISD::FCEIL: 2155 case ISD::FCOS: 2156 case ISD::FEXP: 2157 case ISD::FEXP2: 2158 case ISD::FFLOOR: 2159 case ISD::FLOG: 2160 case ISD::FLOG10: 2161 case ISD::FLOG2: 2162 case ISD::FNEARBYINT: 2163 case ISD::FNEG: 2164 case ISD::FRINT: 2165 case ISD::FROUND: 2166 case ISD::FSIN: 2167 case ISD::FSQRT: 2168 case ISD::FTRUNC: 2169 Res = WidenVecRes_Unary(N); 2170 break; 2171 case ISD::FMA: 2172 Res = WidenVecRes_Ternary(N); 2173 break; 2174 } 2175 2176 // If Res is null, the sub-method took care of registering the result. 2177 if (Res.getNode()) 2178 SetWidenedVector(SDValue(N, ResNo), Res); 2179 } 2180 2181 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { 2182 // Ternary op widening. 2183 SDLoc dl(N); 2184 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2185 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2186 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2187 SDValue InOp3 = GetWidenedVector(N->getOperand(2)); 2188 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3); 2189 } 2190 2191 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 2192 // Binary op widening. 2193 SDLoc dl(N); 2194 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2195 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2196 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2197 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags()); 2198 } 2199 2200 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) { 2201 // Binary op widening for operations that can trap. 2202 unsigned Opcode = N->getOpcode(); 2203 SDLoc dl(N); 2204 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2205 EVT WidenEltVT = WidenVT.getVectorElementType(); 2206 EVT VT = WidenVT; 2207 unsigned NumElts = VT.getVectorNumElements(); 2208 const SDNodeFlags *Flags = N->getFlags(); 2209 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 2210 NumElts = NumElts / 2; 2211 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 2212 } 2213 2214 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) { 2215 // Operation doesn't trap so just widen as normal. 2216 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2217 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2218 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags); 2219 } 2220 2221 // No legal vector version so unroll the vector operation and then widen. 2222 if (NumElts == 1) 2223 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 2224 2225 // Since the operation can trap, apply operation on the original vector. 2226 EVT MaxVT = VT; 2227 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2228 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2229 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 2230 2231 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 2232 unsigned ConcatEnd = 0; // Current ConcatOps index. 2233 int Idx = 0; // Current Idx into input vectors. 2234 2235 // NumElts := greatest legal vector size (at most WidenVT) 2236 // while (orig. vector has unhandled elements) { 2237 // take munches of size NumElts from the beginning and add to ConcatOps 2238 // NumElts := next smaller supported vector size or 1 2239 // } 2240 while (CurNumElts != 0) { 2241 while (CurNumElts >= NumElts) { 2242 SDValue EOp1 = DAG.getNode( 2243 ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1, 2244 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 2245 SDValue EOp2 = DAG.getNode( 2246 ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2, 2247 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 2248 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags); 2249 Idx += NumElts; 2250 CurNumElts -= NumElts; 2251 } 2252 do { 2253 NumElts = NumElts / 2; 2254 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 2255 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 2256 2257 if (NumElts == 1) { 2258 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 2259 SDValue EOp1 = DAG.getNode( 2260 ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1, 2261 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 2262 SDValue EOp2 = DAG.getNode( 2263 ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2, 2264 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 2265 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT, 2266 EOp1, EOp2, Flags); 2267 } 2268 CurNumElts = 0; 2269 } 2270 } 2271 2272 // Check to see if we have a single operation with the widen type. 2273 if (ConcatEnd == 1) { 2274 VT = ConcatOps[0].getValueType(); 2275 if (VT == WidenVT) 2276 return ConcatOps[0]; 2277 } 2278 2279 // while (Some element of ConcatOps is not of type MaxVT) { 2280 // From the end of ConcatOps, collect elements of the same type and put 2281 // them into an op of the next larger supported type 2282 // } 2283 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { 2284 Idx = ConcatEnd - 1; 2285 VT = ConcatOps[Idx--].getValueType(); 2286 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) 2287 Idx--; 2288 2289 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1; 2290 EVT NextVT; 2291 do { 2292 NextSize *= 2; 2293 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize); 2294 } while (!TLI.isTypeLegal(NextVT)); 2295 2296 if (!VT.isVector()) { 2297 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT 2298 SDValue VecOp = DAG.getUNDEF(NextVT); 2299 unsigned NumToInsert = ConcatEnd - Idx - 1; 2300 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) { 2301 VecOp = DAG.getNode( 2302 ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx], 2303 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 2304 } 2305 ConcatOps[Idx+1] = VecOp; 2306 ConcatEnd = Idx + 2; 2307 } else { 2308 // Vector type, create a CONCAT_VECTORS of type NextVT 2309 SDValue undefVec = DAG.getUNDEF(VT); 2310 unsigned OpsToConcat = NextSize/VT.getVectorNumElements(); 2311 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat); 2312 unsigned RealVals = ConcatEnd - Idx - 1; 2313 unsigned SubConcatEnd = 0; 2314 unsigned SubConcatIdx = Idx + 1; 2315 while (SubConcatEnd < RealVals) 2316 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx]; 2317 while (SubConcatEnd < OpsToConcat) 2318 SubConcatOps[SubConcatEnd++] = undefVec; 2319 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl, 2320 NextVT, SubConcatOps); 2321 ConcatEnd = SubConcatIdx + 1; 2322 } 2323 } 2324 2325 // Check to see if we have a single operation with the widen type. 2326 if (ConcatEnd == 1) { 2327 VT = ConcatOps[0].getValueType(); 2328 if (VT == WidenVT) 2329 return ConcatOps[0]; 2330 } 2331 2332 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT 2333 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements(); 2334 if (NumOps != ConcatEnd ) { 2335 SDValue UndefVal = DAG.getUNDEF(MaxVT); 2336 for (unsigned j = ConcatEnd; j < NumOps; ++j) 2337 ConcatOps[j] = UndefVal; 2338 } 2339 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 2340 makeArrayRef(ConcatOps.data(), NumOps)); 2341 } 2342 2343 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 2344 SDValue InOp = N->getOperand(0); 2345 SDLoc DL(N); 2346 2347 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2348 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2349 2350 EVT InVT = InOp.getValueType(); 2351 EVT InEltVT = InVT.getVectorElementType(); 2352 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 2353 2354 unsigned Opcode = N->getOpcode(); 2355 unsigned InVTNumElts = InVT.getVectorNumElements(); 2356 const SDNodeFlags *Flags = N->getFlags(); 2357 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 2358 InOp = GetWidenedVector(N->getOperand(0)); 2359 InVT = InOp.getValueType(); 2360 InVTNumElts = InVT.getVectorNumElements(); 2361 if (InVTNumElts == WidenNumElts) { 2362 if (N->getNumOperands() == 1) 2363 return DAG.getNode(Opcode, DL, WidenVT, InOp); 2364 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags); 2365 } 2366 } 2367 2368 if (TLI.isTypeLegal(InWidenVT)) { 2369 // Because the result and the input are different vector types, widening 2370 // the result could create a legal type but widening the input might make 2371 // it an illegal type that might lead to repeatedly splitting the input 2372 // and then widening it. To avoid this, we widen the input only if 2373 // it results in a legal type. 2374 if (WidenNumElts % InVTNumElts == 0) { 2375 // Widen the input and call convert on the widened input vector. 2376 unsigned NumConcat = WidenNumElts/InVTNumElts; 2377 SmallVector<SDValue, 16> Ops(NumConcat); 2378 Ops[0] = InOp; 2379 SDValue UndefVal = DAG.getUNDEF(InVT); 2380 for (unsigned i = 1; i != NumConcat; ++i) 2381 Ops[i] = UndefVal; 2382 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops); 2383 if (N->getNumOperands() == 1) 2384 return DAG.getNode(Opcode, DL, WidenVT, InVec); 2385 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags); 2386 } 2387 2388 if (InVTNumElts % WidenNumElts == 0) { 2389 SDValue InVal = DAG.getNode( 2390 ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp, 2391 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 2392 // Extract the input and convert the shorten input vector. 2393 if (N->getNumOperands() == 1) 2394 return DAG.getNode(Opcode, DL, WidenVT, InVal); 2395 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags); 2396 } 2397 } 2398 2399 // Otherwise unroll into some nasty scalar code and rebuild the vector. 2400 SmallVector<SDValue, 16> Ops(WidenNumElts); 2401 EVT EltVT = WidenVT.getVectorElementType(); 2402 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 2403 unsigned i; 2404 for (i=0; i < MinElts; ++i) { 2405 SDValue Val = DAG.getNode( 2406 ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 2407 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 2408 if (N->getNumOperands() == 1) 2409 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val); 2410 else 2411 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags); 2412 } 2413 2414 SDValue UndefVal = DAG.getUNDEF(EltVT); 2415 for (; i < WidenNumElts; ++i) 2416 Ops[i] = UndefVal; 2417 2418 return DAG.getNode(ISD::BUILD_VECTOR, DL, WidenVT, Ops); 2419 } 2420 2421 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) { 2422 unsigned Opcode = N->getOpcode(); 2423 SDValue InOp = N->getOperand(0); 2424 SDLoc DL(N); 2425 2426 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2427 EVT WidenSVT = WidenVT.getVectorElementType(); 2428 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2429 2430 EVT InVT = InOp.getValueType(); 2431 EVT InSVT = InVT.getVectorElementType(); 2432 unsigned InVTNumElts = InVT.getVectorNumElements(); 2433 2434 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 2435 InOp = GetWidenedVector(InOp); 2436 InVT = InOp.getValueType(); 2437 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) { 2438 switch (Opcode) { 2439 case ISD::ANY_EXTEND_VECTOR_INREG: 2440 return DAG.getAnyExtendVectorInReg(InOp, DL, WidenVT); 2441 case ISD::SIGN_EXTEND_VECTOR_INREG: 2442 return DAG.getSignExtendVectorInReg(InOp, DL, WidenVT); 2443 case ISD::ZERO_EXTEND_VECTOR_INREG: 2444 return DAG.getZeroExtendVectorInReg(InOp, DL, WidenVT); 2445 } 2446 } 2447 } 2448 2449 // Unroll, extend the scalars and rebuild the vector. 2450 SmallVector<SDValue, 16> Ops; 2451 for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) { 2452 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp, 2453 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 2454 switch (Opcode) { 2455 case ISD::ANY_EXTEND_VECTOR_INREG: 2456 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val); 2457 break; 2458 case ISD::SIGN_EXTEND_VECTOR_INREG: 2459 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val); 2460 break; 2461 case ISD::ZERO_EXTEND_VECTOR_INREG: 2462 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val); 2463 break; 2464 default: 2465 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected"); 2466 } 2467 Ops.push_back(Val); 2468 } 2469 2470 while (Ops.size() != WidenNumElts) 2471 Ops.push_back(DAG.getUNDEF(WidenSVT)); 2472 2473 return DAG.getNode(ISD::BUILD_VECTOR, DL, WidenVT, Ops); 2474 } 2475 2476 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) { 2477 // If this is an FCOPYSIGN with same input types, we can treat it as a 2478 // normal (can trap) binary op. 2479 if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType()) 2480 return WidenVecRes_BinaryCanTrap(N); 2481 2482 // If the types are different, fall back to unrolling. 2483 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2484 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 2485 } 2486 2487 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) { 2488 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2489 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2490 SDValue ShOp = N->getOperand(1); 2491 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 2492 } 2493 2494 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 2495 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2496 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2497 SDValue ShOp = N->getOperand(1); 2498 2499 EVT ShVT = ShOp.getValueType(); 2500 if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) { 2501 ShOp = GetWidenedVector(ShOp); 2502 ShVT = ShOp.getValueType(); 2503 } 2504 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), 2505 ShVT.getVectorElementType(), 2506 WidenVT.getVectorNumElements()); 2507 if (ShVT != ShWidenVT) 2508 ShOp = ModifyToType(ShOp, ShWidenVT); 2509 2510 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 2511 } 2512 2513 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 2514 // Unary op widening. 2515 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2516 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2517 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp); 2518 } 2519 2520 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 2521 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2522 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 2523 cast<VTSDNode>(N->getOperand(1))->getVT() 2524 .getVectorElementType(), 2525 WidenVT.getVectorNumElements()); 2526 SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 2527 return DAG.getNode(N->getOpcode(), SDLoc(N), 2528 WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 2529 } 2530 2531 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { 2532 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); 2533 return GetWidenedVector(WidenVec); 2534 } 2535 2536 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { 2537 SDValue InOp = N->getOperand(0); 2538 EVT InVT = InOp.getValueType(); 2539 EVT VT = N->getValueType(0); 2540 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2541 SDLoc dl(N); 2542 2543 switch (getTypeAction(InVT)) { 2544 case TargetLowering::TypeLegal: 2545 break; 2546 case TargetLowering::TypePromoteInteger: 2547 // If the incoming type is a vector that is being promoted, then 2548 // we know that the elements are arranged differently and that we 2549 // must perform the conversion using a stack slot. 2550 if (InVT.isVector()) 2551 break; 2552 2553 // If the InOp is promoted to the same size, convert it. Otherwise, 2554 // fall out of the switch and widen the promoted input. 2555 InOp = GetPromotedInteger(InOp); 2556 InVT = InOp.getValueType(); 2557 if (WidenVT.bitsEq(InVT)) 2558 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 2559 break; 2560 case TargetLowering::TypeSoftenFloat: 2561 case TargetLowering::TypePromoteFloat: 2562 case TargetLowering::TypeExpandInteger: 2563 case TargetLowering::TypeExpandFloat: 2564 case TargetLowering::TypeScalarizeVector: 2565 case TargetLowering::TypeSplitVector: 2566 break; 2567 case TargetLowering::TypeWidenVector: 2568 // If the InOp is widened to the same size, convert it. Otherwise, fall 2569 // out of the switch and widen the widened input. 2570 InOp = GetWidenedVector(InOp); 2571 InVT = InOp.getValueType(); 2572 if (WidenVT.bitsEq(InVT)) 2573 // The input widens to the same size. Convert to the widen value. 2574 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 2575 break; 2576 } 2577 2578 unsigned WidenSize = WidenVT.getSizeInBits(); 2579 unsigned InSize = InVT.getSizeInBits(); 2580 // x86mmx is not an acceptable vector element type, so don't try. 2581 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) { 2582 // Determine new input vector type. The new input vector type will use 2583 // the same element type (if its a vector) or use the input type as a 2584 // vector. It is the same size as the type to widen to. 2585 EVT NewInVT; 2586 unsigned NewNumElts = WidenSize / InSize; 2587 if (InVT.isVector()) { 2588 EVT InEltVT = InVT.getVectorElementType(); 2589 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, 2590 WidenSize / InEltVT.getSizeInBits()); 2591 } else { 2592 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 2593 } 2594 2595 if (TLI.isTypeLegal(NewInVT)) { 2596 // Because the result and the input are different vector types, widening 2597 // the result could create a legal type but widening the input might make 2598 // it an illegal type that might lead to repeatedly splitting the input 2599 // and then widening it. To avoid this, we widen the input only if 2600 // it results in a legal type. 2601 SmallVector<SDValue, 16> Ops(NewNumElts); 2602 SDValue UndefVal = DAG.getUNDEF(InVT); 2603 Ops[0] = InOp; 2604 for (unsigned i = 1; i < NewNumElts; ++i) 2605 Ops[i] = UndefVal; 2606 2607 SDValue NewVec; 2608 if (InVT.isVector()) 2609 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops); 2610 else 2611 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, NewInVT, Ops); 2612 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec); 2613 } 2614 } 2615 2616 return CreateStackStoreLoad(InOp, WidenVT); 2617 } 2618 2619 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 2620 SDLoc dl(N); 2621 // Build a vector with undefined for the new nodes. 2622 EVT VT = N->getValueType(0); 2623 2624 // Integer BUILD_VECTOR operands may be larger than the node's vector element 2625 // type. The UNDEFs need to have the same type as the existing operands. 2626 EVT EltVT = N->getOperand(0).getValueType(); 2627 unsigned NumElts = VT.getVectorNumElements(); 2628 2629 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2630 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2631 2632 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 2633 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!"); 2634 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT)); 2635 2636 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, NewOps); 2637 } 2638 2639 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 2640 EVT InVT = N->getOperand(0).getValueType(); 2641 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2642 SDLoc dl(N); 2643 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2644 unsigned NumInElts = InVT.getVectorNumElements(); 2645 unsigned NumOperands = N->getNumOperands(); 2646 2647 bool InputWidened = false; // Indicates we need to widen the input. 2648 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) { 2649 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 2650 // Add undef vectors to widen to correct length. 2651 unsigned NumConcat = WidenVT.getVectorNumElements() / 2652 InVT.getVectorNumElements(); 2653 SDValue UndefVal = DAG.getUNDEF(InVT); 2654 SmallVector<SDValue, 16> Ops(NumConcat); 2655 for (unsigned i=0; i < NumOperands; ++i) 2656 Ops[i] = N->getOperand(i); 2657 for (unsigned i = NumOperands; i != NumConcat; ++i) 2658 Ops[i] = UndefVal; 2659 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops); 2660 } 2661 } else { 2662 InputWidened = true; 2663 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 2664 // The inputs and the result are widen to the same value. 2665 unsigned i; 2666 for (i=1; i < NumOperands; ++i) 2667 if (!N->getOperand(i).isUndef()) 2668 break; 2669 2670 if (i == NumOperands) 2671 // Everything but the first operand is an UNDEF so just return the 2672 // widened first operand. 2673 return GetWidenedVector(N->getOperand(0)); 2674 2675 if (NumOperands == 2) { 2676 // Replace concat of two operands with a shuffle. 2677 SmallVector<int, 16> MaskOps(WidenNumElts, -1); 2678 for (unsigned i = 0; i < NumInElts; ++i) { 2679 MaskOps[i] = i; 2680 MaskOps[i + NumInElts] = i + WidenNumElts; 2681 } 2682 return DAG.getVectorShuffle(WidenVT, dl, 2683 GetWidenedVector(N->getOperand(0)), 2684 GetWidenedVector(N->getOperand(1)), 2685 MaskOps); 2686 } 2687 } 2688 } 2689 2690 // Fall back to use extracts and build vector. 2691 EVT EltVT = WidenVT.getVectorElementType(); 2692 SmallVector<SDValue, 16> Ops(WidenNumElts); 2693 unsigned Idx = 0; 2694 for (unsigned i=0; i < NumOperands; ++i) { 2695 SDValue InOp = N->getOperand(i); 2696 if (InputWidened) 2697 InOp = GetWidenedVector(InOp); 2698 for (unsigned j=0; j < NumInElts; ++j) 2699 Ops[Idx++] = DAG.getNode( 2700 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2701 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 2702 } 2703 SDValue UndefVal = DAG.getUNDEF(EltVT); 2704 for (; Idx < WidenNumElts; ++Idx) 2705 Ops[Idx] = UndefVal; 2706 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2707 } 2708 2709 SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 2710 SDLoc dl(N); 2711 SDValue InOp = N->getOperand(0); 2712 SDValue RndOp = N->getOperand(3); 2713 SDValue SatOp = N->getOperand(4); 2714 2715 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2716 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2717 2718 EVT InVT = InOp.getValueType(); 2719 EVT InEltVT = InVT.getVectorElementType(); 2720 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 2721 2722 SDValue DTyOp = DAG.getValueType(WidenVT); 2723 SDValue STyOp = DAG.getValueType(InWidenVT); 2724 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 2725 2726 unsigned InVTNumElts = InVT.getVectorNumElements(); 2727 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 2728 InOp = GetWidenedVector(InOp); 2729 InVT = InOp.getValueType(); 2730 InVTNumElts = InVT.getVectorNumElements(); 2731 if (InVTNumElts == WidenNumElts) 2732 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2733 SatOp, CvtCode); 2734 } 2735 2736 if (TLI.isTypeLegal(InWidenVT)) { 2737 // Because the result and the input are different vector types, widening 2738 // the result could create a legal type but widening the input might make 2739 // it an illegal type that might lead to repeatedly splitting the input 2740 // and then widening it. To avoid this, we widen the input only if 2741 // it results in a legal type. 2742 if (WidenNumElts % InVTNumElts == 0) { 2743 // Widen the input and call convert on the widened input vector. 2744 unsigned NumConcat = WidenNumElts/InVTNumElts; 2745 SmallVector<SDValue, 16> Ops(NumConcat); 2746 Ops[0] = InOp; 2747 SDValue UndefVal = DAG.getUNDEF(InVT); 2748 for (unsigned i = 1; i != NumConcat; ++i) 2749 Ops[i] = UndefVal; 2750 2751 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, Ops); 2752 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2753 SatOp, CvtCode); 2754 } 2755 2756 if (InVTNumElts % WidenNumElts == 0) { 2757 // Extract the input and convert the shorten input vector. 2758 InOp = DAG.getNode( 2759 ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 2760 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 2761 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2762 SatOp, CvtCode); 2763 } 2764 } 2765 2766 // Otherwise unroll into some nasty scalar code and rebuild the vector. 2767 SmallVector<SDValue, 16> Ops(WidenNumElts); 2768 EVT EltVT = WidenVT.getVectorElementType(); 2769 DTyOp = DAG.getValueType(EltVT); 2770 STyOp = DAG.getValueType(InEltVT); 2771 2772 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 2773 unsigned i; 2774 for (i=0; i < MinElts; ++i) { 2775 SDValue ExtVal = DAG.getNode( 2776 ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 2777 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 2778 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 2779 SatOp, CvtCode); 2780 } 2781 2782 SDValue UndefVal = DAG.getUNDEF(EltVT); 2783 for (; i < WidenNumElts; ++i) 2784 Ops[i] = UndefVal; 2785 2786 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2787 } 2788 2789 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 2790 EVT VT = N->getValueType(0); 2791 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2792 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2793 SDValue InOp = N->getOperand(0); 2794 SDValue Idx = N->getOperand(1); 2795 SDLoc dl(N); 2796 2797 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2798 InOp = GetWidenedVector(InOp); 2799 2800 EVT InVT = InOp.getValueType(); 2801 2802 // Check if we can just return the input vector after widening. 2803 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2804 if (IdxVal == 0 && InVT == WidenVT) 2805 return InOp; 2806 2807 // Check if we can extract from the vector. 2808 unsigned InNumElts = InVT.getVectorNumElements(); 2809 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 2810 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 2811 2812 // We could try widening the input to the right length but for now, extract 2813 // the original elements, fill the rest with undefs and build a vector. 2814 SmallVector<SDValue, 16> Ops(WidenNumElts); 2815 EVT EltVT = VT.getVectorElementType(); 2816 unsigned NumElts = VT.getVectorNumElements(); 2817 unsigned i; 2818 for (i=0; i < NumElts; ++i) 2819 Ops[i] = 2820 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2821 DAG.getConstant(IdxVal + i, dl, 2822 TLI.getVectorIdxTy(DAG.getDataLayout()))); 2823 2824 SDValue UndefVal = DAG.getUNDEF(EltVT); 2825 for (; i < WidenNumElts; ++i) 2826 Ops[i] = UndefVal; 2827 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 2828 } 2829 2830 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 2831 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2832 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), 2833 InOp.getValueType(), InOp, 2834 N->getOperand(1), N->getOperand(2)); 2835 } 2836 2837 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 2838 LoadSDNode *LD = cast<LoadSDNode>(N); 2839 ISD::LoadExtType ExtType = LD->getExtensionType(); 2840 2841 SDValue Result; 2842 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 2843 if (ExtType != ISD::NON_EXTLOAD) 2844 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); 2845 else 2846 Result = GenWidenVectorLoads(LdChain, LD); 2847 2848 // If we generate a single load, we can use that for the chain. Otherwise, 2849 // build a factor node to remember the multiple loads are independent and 2850 // chain to that. 2851 SDValue NewChain; 2852 if (LdChain.size() == 1) 2853 NewChain = LdChain[0]; 2854 else 2855 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain); 2856 2857 // Modified the chain - switch anything that used the old chain to use 2858 // the new one. 2859 ReplaceValueWith(SDValue(N, 1), NewChain); 2860 2861 return Result; 2862 } 2863 2864 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) { 2865 2866 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0)); 2867 SDValue Mask = N->getMask(); 2868 EVT MaskVT = Mask.getValueType(); 2869 SDValue Src0 = GetWidenedVector(N->getSrc0()); 2870 ISD::LoadExtType ExtType = N->getExtensionType(); 2871 SDLoc dl(N); 2872 2873 if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector) 2874 Mask = GetWidenedVector(Mask); 2875 else { 2876 EVT BoolVT = getSetCCResultType(WidenVT); 2877 2878 // We can't use ModifyToType() because we should fill the mask with 2879 // zeroes 2880 unsigned WidenNumElts = BoolVT.getVectorNumElements(); 2881 unsigned MaskNumElts = MaskVT.getVectorNumElements(); 2882 2883 unsigned NumConcat = WidenNumElts / MaskNumElts; 2884 SmallVector<SDValue, 16> Ops(NumConcat); 2885 SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT); 2886 Ops[0] = Mask; 2887 for (unsigned i = 1; i != NumConcat; ++i) 2888 Ops[i] = ZeroVal; 2889 2890 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops); 2891 } 2892 2893 SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(), 2894 Mask, Src0, N->getMemoryVT(), 2895 N->getMemOperand(), ExtType); 2896 // Legalize the chain result - switch anything that used the old chain to 2897 // use the new one. 2898 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2899 return Res; 2900 } 2901 2902 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) { 2903 2904 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2905 SDValue Mask = N->getMask(); 2906 SDValue Src0 = GetWidenedVector(N->getValue()); 2907 unsigned NumElts = WideVT.getVectorNumElements(); 2908 SDLoc dl(N); 2909 2910 // The mask should be widened as well 2911 Mask = WidenTargetBoolean(Mask, WideVT, true); 2912 2913 // Widen the Index operand 2914 SDValue Index = N->getIndex(); 2915 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(), 2916 Index.getValueType().getScalarType(), 2917 NumElts); 2918 Index = ModifyToType(Index, WideIndexVT); 2919 SDValue Ops[] = { N->getChain(), Src0, Mask, N->getBasePtr(), Index }; 2920 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other), 2921 N->getMemoryVT(), dl, Ops, 2922 N->getMemOperand()); 2923 2924 // Legalize the chain result - switch anything that used the old chain to 2925 // use the new one. 2926 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2927 return Res; 2928 } 2929 2930 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 2931 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2932 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), 2933 WidenVT, N->getOperand(0)); 2934 } 2935 2936 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 2937 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2938 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2939 2940 SDValue Cond1 = N->getOperand(0); 2941 EVT CondVT = Cond1.getValueType(); 2942 if (CondVT.isVector()) { 2943 EVT CondEltVT = CondVT.getVectorElementType(); 2944 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), 2945 CondEltVT, WidenNumElts); 2946 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector) 2947 Cond1 = GetWidenedVector(Cond1); 2948 2949 // If we have to split the condition there is no point in widening the 2950 // select. This would result in an cycle of widening the select -> 2951 // widening the condition operand -> splitting the condition operand -> 2952 // splitting the select -> widening the select. Instead split this select 2953 // further and widen the resulting type. 2954 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) { 2955 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0); 2956 SDValue Res = ModifyToType(SplitSelect, WidenVT); 2957 return Res; 2958 } 2959 2960 if (Cond1.getValueType() != CondWidenVT) 2961 Cond1 = ModifyToType(Cond1, CondWidenVT); 2962 } 2963 2964 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 2965 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 2966 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 2967 return DAG.getNode(N->getOpcode(), SDLoc(N), 2968 WidenVT, Cond1, InOp1, InOp2); 2969 } 2970 2971 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 2972 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 2973 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 2974 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 2975 InOp1.getValueType(), N->getOperand(0), 2976 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 2977 } 2978 2979 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { 2980 assert(N->getValueType(0).isVector() == 2981 N->getOperand(0).getValueType().isVector() && 2982 "Scalar/Vector type mismatch"); 2983 if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N); 2984 2985 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2986 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2987 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2988 return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT, 2989 InOp1, InOp2, N->getOperand(2)); 2990 } 2991 2992 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 2993 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2994 return DAG.getUNDEF(WidenVT); 2995 } 2996 2997 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 2998 EVT VT = N->getValueType(0); 2999 SDLoc dl(N); 3000 3001 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3002 unsigned NumElts = VT.getVectorNumElements(); 3003 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3004 3005 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 3006 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 3007 3008 // Adjust mask based on new input vector length. 3009 SmallVector<int, 16> NewMask; 3010 for (unsigned i = 0; i != NumElts; ++i) { 3011 int Idx = N->getMaskElt(i); 3012 if (Idx < (int)NumElts) 3013 NewMask.push_back(Idx); 3014 else 3015 NewMask.push_back(Idx - NumElts + WidenNumElts); 3016 } 3017 for (unsigned i = NumElts; i != WidenNumElts; ++i) 3018 NewMask.push_back(-1); 3019 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask); 3020 } 3021 3022 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 3023 assert(N->getValueType(0).isVector() && 3024 N->getOperand(0).getValueType().isVector() && 3025 "Operands must be vectors"); 3026 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3027 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3028 3029 SDValue InOp1 = N->getOperand(0); 3030 EVT InVT = InOp1.getValueType(); 3031 assert(InVT.isVector() && "can not widen non-vector type"); 3032 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), 3033 InVT.getVectorElementType(), WidenNumElts); 3034 3035 // The input and output types often differ here, and it could be that while 3036 // we'd prefer to widen the result type, the input operands have been split. 3037 // In this case, we also need to split the result of this node as well. 3038 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) { 3039 SDValue SplitVSetCC = SplitVecOp_VSETCC(N); 3040 SDValue Res = ModifyToType(SplitVSetCC, WidenVT); 3041 return Res; 3042 } 3043 3044 InOp1 = GetWidenedVector(InOp1); 3045 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 3046 3047 // Assume that the input and output will be widen appropriately. If not, 3048 // we will have to unroll it at some point. 3049 assert(InOp1.getValueType() == WidenInVT && 3050 InOp2.getValueType() == WidenInVT && 3051 "Input not widened to expected type!"); 3052 (void)WidenInVT; 3053 return DAG.getNode(ISD::SETCC, SDLoc(N), 3054 WidenVT, InOp1, InOp2, N->getOperand(2)); 3055 } 3056 3057 3058 //===----------------------------------------------------------------------===// 3059 // Widen Vector Operand 3060 //===----------------------------------------------------------------------===// 3061 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { 3062 DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; 3063 N->dump(&DAG); 3064 dbgs() << "\n"); 3065 SDValue Res = SDValue(); 3066 3067 // See if the target wants to custom widen this node. 3068 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 3069 return false; 3070 3071 switch (N->getOpcode()) { 3072 default: 3073 #ifndef NDEBUG 3074 dbgs() << "WidenVectorOperand op #" << OpNo << ": "; 3075 N->dump(&DAG); 3076 dbgs() << "\n"; 3077 #endif 3078 llvm_unreachable("Do not know how to widen this operator's operand!"); 3079 3080 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break; 3081 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 3082 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 3083 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 3084 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 3085 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break; 3086 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break; 3087 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; 3088 case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break; 3089 3090 case ISD::ANY_EXTEND: 3091 case ISD::SIGN_EXTEND: 3092 case ISD::ZERO_EXTEND: 3093 Res = WidenVecOp_EXTEND(N); 3094 break; 3095 3096 case ISD::FP_EXTEND: 3097 case ISD::FP_TO_SINT: 3098 case ISD::FP_TO_UINT: 3099 case ISD::SINT_TO_FP: 3100 case ISD::UINT_TO_FP: 3101 case ISD::TRUNCATE: 3102 Res = WidenVecOp_Convert(N); 3103 break; 3104 } 3105 3106 // If Res is null, the sub-method took care of registering the result. 3107 if (!Res.getNode()) return false; 3108 3109 // If the result is N, the sub-method updated N in place. Tell the legalizer 3110 // core about this. 3111 if (Res.getNode() == N) 3112 return true; 3113 3114 3115 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 3116 "Invalid operand expansion"); 3117 3118 ReplaceValueWith(SDValue(N, 0), Res); 3119 return false; 3120 } 3121 3122 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) { 3123 SDLoc DL(N); 3124 EVT VT = N->getValueType(0); 3125 3126 SDValue InOp = N->getOperand(0); 3127 // If some legalization strategy other than widening is used on the operand, 3128 // we can't safely assume that just extending the low lanes is the correct 3129 // transformation. 3130 if (getTypeAction(InOp.getValueType()) != TargetLowering::TypeWidenVector) 3131 return WidenVecOp_Convert(N); 3132 InOp = GetWidenedVector(InOp); 3133 assert(VT.getVectorNumElements() < 3134 InOp.getValueType().getVectorNumElements() && 3135 "Input wasn't widened!"); 3136 3137 // We may need to further widen the operand until it has the same total 3138 // vector size as the result. 3139 EVT InVT = InOp.getValueType(); 3140 if (InVT.getSizeInBits() != VT.getSizeInBits()) { 3141 EVT InEltVT = InVT.getVectorElementType(); 3142 for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) { 3143 EVT FixedVT = (MVT::SimpleValueType)i; 3144 EVT FixedEltVT = FixedVT.getVectorElementType(); 3145 if (TLI.isTypeLegal(FixedVT) && 3146 FixedVT.getSizeInBits() == VT.getSizeInBits() && 3147 FixedEltVT == InEltVT) { 3148 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() && 3149 "Not enough elements in the fixed type for the operand!"); 3150 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() && 3151 "We can't have the same type as we started with!"); 3152 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements()) 3153 InOp = DAG.getNode( 3154 ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp, 3155 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3156 else 3157 InOp = DAG.getNode( 3158 ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp, 3159 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3160 break; 3161 } 3162 } 3163 InVT = InOp.getValueType(); 3164 if (InVT.getSizeInBits() != VT.getSizeInBits()) 3165 // We couldn't find a legal vector type that was a widening of the input 3166 // and could be extended in-register to the result type, so we have to 3167 // scalarize. 3168 return WidenVecOp_Convert(N); 3169 } 3170 3171 // Use special DAG nodes to represent the operation of extending the 3172 // low lanes. 3173 switch (N->getOpcode()) { 3174 default: 3175 llvm_unreachable("Extend legalization on on extend operation!"); 3176 case ISD::ANY_EXTEND: 3177 return DAG.getAnyExtendVectorInReg(InOp, DL, VT); 3178 case ISD::SIGN_EXTEND: 3179 return DAG.getSignExtendVectorInReg(InOp, DL, VT); 3180 case ISD::ZERO_EXTEND: 3181 return DAG.getZeroExtendVectorInReg(InOp, DL, VT); 3182 } 3183 } 3184 3185 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) { 3186 // The result (and first input) is legal, but the second input is illegal. 3187 // We can't do much to fix that, so just unroll and let the extracts off of 3188 // the second input be widened as needed later. 3189 return DAG.UnrollVectorOp(N); 3190 } 3191 3192 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 3193 // Since the result is legal and the input is illegal, it is unlikely that we 3194 // can fix the input to a legal type so unroll the convert into some scalar 3195 // code and create a nasty build vector. 3196 EVT VT = N->getValueType(0); 3197 EVT EltVT = VT.getVectorElementType(); 3198 SDLoc dl(N); 3199 unsigned NumElts = VT.getVectorNumElements(); 3200 SDValue InOp = N->getOperand(0); 3201 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 3202 InOp = GetWidenedVector(InOp); 3203 EVT InVT = InOp.getValueType(); 3204 EVT InEltVT = InVT.getVectorElementType(); 3205 3206 unsigned Opcode = N->getOpcode(); 3207 SmallVector<SDValue, 16> Ops(NumElts); 3208 for (unsigned i=0; i < NumElts; ++i) 3209 Ops[i] = DAG.getNode( 3210 Opcode, dl, EltVT, 3211 DAG.getNode( 3212 ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 3213 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())))); 3214 3215 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); 3216 } 3217 3218 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { 3219 EVT VT = N->getValueType(0); 3220 SDValue InOp = GetWidenedVector(N->getOperand(0)); 3221 EVT InWidenVT = InOp.getValueType(); 3222 SDLoc dl(N); 3223 3224 // Check if we can convert between two legal vector types and extract. 3225 unsigned InWidenSize = InWidenVT.getSizeInBits(); 3226 unsigned Size = VT.getSizeInBits(); 3227 // x86mmx is not an acceptable vector element type, so don't try. 3228 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) { 3229 unsigned NewNumElts = InWidenSize / Size; 3230 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 3231 if (TLI.isTypeLegal(NewVT)) { 3232 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 3233 return DAG.getNode( 3234 ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 3235 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3236 } 3237 } 3238 3239 return CreateStackStoreLoad(InOp, VT); 3240 } 3241 3242 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 3243 // If the input vector is not legal, it is likely that we will not find a 3244 // legal vector of the same size. Replace the concatenate vector with a 3245 // nasty build vector. 3246 EVT VT = N->getValueType(0); 3247 EVT EltVT = VT.getVectorElementType(); 3248 SDLoc dl(N); 3249 unsigned NumElts = VT.getVectorNumElements(); 3250 SmallVector<SDValue, 16> Ops(NumElts); 3251 3252 EVT InVT = N->getOperand(0).getValueType(); 3253 unsigned NumInElts = InVT.getVectorNumElements(); 3254 3255 unsigned Idx = 0; 3256 unsigned NumOperands = N->getNumOperands(); 3257 for (unsigned i=0; i < NumOperands; ++i) { 3258 SDValue InOp = N->getOperand(i); 3259 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 3260 InOp = GetWidenedVector(InOp); 3261 for (unsigned j=0; j < NumInElts; ++j) 3262 Ops[Idx++] = DAG.getNode( 3263 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 3264 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3265 } 3266 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); 3267 } 3268 3269 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 3270 SDValue InOp = GetWidenedVector(N->getOperand(0)); 3271 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), 3272 N->getValueType(0), InOp, N->getOperand(1)); 3273 } 3274 3275 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 3276 SDValue InOp = GetWidenedVector(N->getOperand(0)); 3277 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 3278 N->getValueType(0), InOp, N->getOperand(1)); 3279 } 3280 3281 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 3282 // We have to widen the value, but we want only to store the original 3283 // vector type. 3284 StoreSDNode *ST = cast<StoreSDNode>(N); 3285 3286 SmallVector<SDValue, 16> StChain; 3287 if (ST->isTruncatingStore()) 3288 GenWidenVectorTruncStores(StChain, ST); 3289 else 3290 GenWidenVectorStores(StChain, ST); 3291 3292 if (StChain.size() == 1) 3293 return StChain[0]; 3294 else 3295 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain); 3296 } 3297 3298 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) { 3299 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N); 3300 SDValue Mask = MST->getMask(); 3301 EVT MaskVT = Mask.getValueType(); 3302 SDValue StVal = MST->getValue(); 3303 // Widen the value 3304 SDValue WideVal = GetWidenedVector(StVal); 3305 SDLoc dl(N); 3306 3307 if (OpNo == 2 || getTypeAction(MaskVT) == TargetLowering::TypeWidenVector) 3308 Mask = GetWidenedVector(Mask); 3309 else { 3310 // The mask should be widened as well. 3311 EVT BoolVT = getSetCCResultType(WideVal.getValueType()); 3312 // We can't use ModifyToType() because we should fill the mask with 3313 // zeroes. 3314 unsigned WidenNumElts = BoolVT.getVectorNumElements(); 3315 unsigned MaskNumElts = MaskVT.getVectorNumElements(); 3316 3317 unsigned NumConcat = WidenNumElts / MaskNumElts; 3318 SmallVector<SDValue, 16> Ops(NumConcat); 3319 SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT); 3320 Ops[0] = Mask; 3321 for (unsigned i = 1; i != NumConcat; ++i) 3322 Ops[i] = ZeroVal; 3323 3324 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops); 3325 } 3326 assert(Mask.getValueType().getVectorNumElements() == 3327 WideVal.getValueType().getVectorNumElements() && 3328 "Mask and data vectors should have the same number of elements"); 3329 return DAG.getMaskedStore(MST->getChain(), dl, WideVal, MST->getBasePtr(), 3330 Mask, MST->getMemoryVT(), MST->getMemOperand(), 3331 false); 3332 } 3333 3334 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) { 3335 assert(OpNo == 1 && "Can widen only data operand of mscatter"); 3336 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N); 3337 SDValue DataOp = MSC->getValue(); 3338 SDValue Mask = MSC->getMask(); 3339 3340 // Widen the value. 3341 SDValue WideVal = GetWidenedVector(DataOp); 3342 EVT WideVT = WideVal.getValueType(); 3343 unsigned NumElts = WideVal.getValueType().getVectorNumElements(); 3344 SDLoc dl(N); 3345 3346 // The mask should be widened as well. 3347 Mask = WidenTargetBoolean(Mask, WideVT, true); 3348 3349 // Widen index. 3350 SDValue Index = MSC->getIndex(); 3351 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(), 3352 Index.getValueType().getScalarType(), 3353 NumElts); 3354 Index = ModifyToType(Index, WideIndexVT); 3355 3356 SDValue Ops[] = {MSC->getChain(), WideVal, Mask, MSC->getBasePtr(), Index}; 3357 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), 3358 MSC->getMemoryVT(), dl, Ops, 3359 MSC->getMemOperand()); 3360 } 3361 3362 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { 3363 SDValue InOp0 = GetWidenedVector(N->getOperand(0)); 3364 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 3365 SDLoc dl(N); 3366 3367 // WARNING: In this code we widen the compare instruction with garbage. 3368 // This garbage may contain denormal floats which may be slow. Is this a real 3369 // concern ? Should we zero the unused lanes if this is a float compare ? 3370 3371 // Get a new SETCC node to compare the newly widened operands. 3372 // Only some of the compared elements are legal. 3373 EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), 3374 InOp0.getValueType()); 3375 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N), 3376 SVT, InOp0, InOp1, N->getOperand(2)); 3377 3378 // Extract the needed results from the result vector. 3379 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), 3380 SVT.getVectorElementType(), 3381 N->getValueType(0).getVectorNumElements()); 3382 SDValue CC = DAG.getNode( 3383 ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC, 3384 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3385 3386 return PromoteTargetBoolean(CC, N->getValueType(0)); 3387 } 3388 3389 3390 //===----------------------------------------------------------------------===// 3391 // Vector Widening Utilities 3392 //===----------------------------------------------------------------------===// 3393 3394 // Utility function to find the type to chop up a widen vector for load/store 3395 // TLI: Target lowering used to determine legal types. 3396 // Width: Width left need to load/store. 3397 // WidenVT: The widen vector type to load to/store from 3398 // Align: If 0, don't allow use of a wider type 3399 // WidenEx: If Align is not 0, the amount additional we can load/store from. 3400 3401 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI, 3402 unsigned Width, EVT WidenVT, 3403 unsigned Align = 0, unsigned WidenEx = 0) { 3404 EVT WidenEltVT = WidenVT.getVectorElementType(); 3405 unsigned WidenWidth = WidenVT.getSizeInBits(); 3406 unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); 3407 unsigned AlignInBits = Align*8; 3408 3409 // If we have one element to load/store, return it. 3410 EVT RetVT = WidenEltVT; 3411 if (Width == WidenEltWidth) 3412 return RetVT; 3413 3414 // See if there is larger legal integer than the element type to load/store. 3415 unsigned VT; 3416 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE; 3417 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) { 3418 EVT MemVT((MVT::SimpleValueType) VT); 3419 unsigned MemVTWidth = MemVT.getSizeInBits(); 3420 if (MemVT.getSizeInBits() <= WidenEltWidth) 3421 break; 3422 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT); 3423 if ((Action == TargetLowering::TypeLegal || 3424 Action == TargetLowering::TypePromoteInteger) && 3425 (WidenWidth % MemVTWidth) == 0 && 3426 isPowerOf2_32(WidenWidth / MemVTWidth) && 3427 (MemVTWidth <= Width || 3428 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 3429 RetVT = MemVT; 3430 break; 3431 } 3432 } 3433 3434 // See if there is a larger vector type to load/store that has the same vector 3435 // element type and is evenly divisible with the WidenVT. 3436 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE; 3437 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) { 3438 EVT MemVT = (MVT::SimpleValueType) VT; 3439 unsigned MemVTWidth = MemVT.getSizeInBits(); 3440 if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() && 3441 (WidenWidth % MemVTWidth) == 0 && 3442 isPowerOf2_32(WidenWidth / MemVTWidth) && 3443 (MemVTWidth <= Width || 3444 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 3445 if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT) 3446 return MemVT; 3447 } 3448 } 3449 3450 return RetVT; 3451 } 3452 3453 // Builds a vector type from scalar loads 3454 // VecTy: Resulting Vector type 3455 // LDOps: Load operators to build a vector type 3456 // [Start,End) the list of loads to use. 3457 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, 3458 SmallVectorImpl<SDValue> &LdOps, 3459 unsigned Start, unsigned End) { 3460 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3461 SDLoc dl(LdOps[Start]); 3462 EVT LdTy = LdOps[Start].getValueType(); 3463 unsigned Width = VecTy.getSizeInBits(); 3464 unsigned NumElts = Width / LdTy.getSizeInBits(); 3465 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts); 3466 3467 unsigned Idx = 1; 3468 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]); 3469 3470 for (unsigned i = Start + 1; i != End; ++i) { 3471 EVT NewLdTy = LdOps[i].getValueType(); 3472 if (NewLdTy != LdTy) { 3473 NumElts = Width / NewLdTy.getSizeInBits(); 3474 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts); 3475 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp); 3476 // Readjust position and vector position based on new load type. 3477 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); 3478 LdTy = NewLdTy; 3479 } 3480 VecOp = DAG.getNode( 3481 ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i], 3482 DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3483 } 3484 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp); 3485 } 3486 3487 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain, 3488 LoadSDNode *LD) { 3489 // The strategy assumes that we can efficiently load power-of-two widths. 3490 // The routine chops the vector into the largest vector loads with the same 3491 // element type or scalar loads and then recombines it to the widen vector 3492 // type. 3493 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 3494 unsigned WidenWidth = WidenVT.getSizeInBits(); 3495 EVT LdVT = LD->getMemoryVT(); 3496 SDLoc dl(LD); 3497 assert(LdVT.isVector() && WidenVT.isVector()); 3498 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 3499 3500 // Load information 3501 SDValue Chain = LD->getChain(); 3502 SDValue BasePtr = LD->getBasePtr(); 3503 unsigned Align = LD->getAlignment(); 3504 bool isVolatile = LD->isVolatile(); 3505 bool isNonTemporal = LD->isNonTemporal(); 3506 bool isInvariant = LD->isInvariant(); 3507 AAMDNodes AAInfo = LD->getAAInfo(); 3508 3509 int LdWidth = LdVT.getSizeInBits(); 3510 int WidthDiff = WidenWidth - LdWidth; 3511 unsigned LdAlign = (isVolatile) ? 0 : Align; // Allow wider loads. 3512 3513 // Find the vector type that can load from. 3514 EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 3515 int NewVTWidth = NewVT.getSizeInBits(); 3516 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(), 3517 isVolatile, isNonTemporal, isInvariant, Align, 3518 AAInfo); 3519 LdChain.push_back(LdOp.getValue(1)); 3520 3521 // Check if we can load the element with one instruction. 3522 if (LdWidth <= NewVTWidth) { 3523 if (!NewVT.isVector()) { 3524 unsigned NumElts = WidenWidth / NewVTWidth; 3525 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 3526 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 3527 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp); 3528 } 3529 if (NewVT == WidenVT) 3530 return LdOp; 3531 3532 assert(WidenWidth % NewVTWidth == 0); 3533 unsigned NumConcat = WidenWidth / NewVTWidth; 3534 SmallVector<SDValue, 16> ConcatOps(NumConcat); 3535 SDValue UndefVal = DAG.getUNDEF(NewVT); 3536 ConcatOps[0] = LdOp; 3537 for (unsigned i = 1; i != NumConcat; ++i) 3538 ConcatOps[i] = UndefVal; 3539 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps); 3540 } 3541 3542 // Load vector by using multiple loads from largest vector to scalar. 3543 SmallVector<SDValue, 16> LdOps; 3544 LdOps.push_back(LdOp); 3545 3546 LdWidth -= NewVTWidth; 3547 unsigned Offset = 0; 3548 3549 while (LdWidth > 0) { 3550 unsigned Increment = NewVTWidth / 8; 3551 Offset += Increment; 3552 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 3553 DAG.getConstant(Increment, dl, BasePtr.getValueType())); 3554 3555 SDValue L; 3556 if (LdWidth < NewVTWidth) { 3557 // The current type we are using is too large. Find a better size. 3558 NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 3559 NewVTWidth = NewVT.getSizeInBits(); 3560 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 3561 LD->getPointerInfo().getWithOffset(Offset), isVolatile, 3562 isNonTemporal, isInvariant, MinAlign(Align, Increment), 3563 AAInfo); 3564 LdChain.push_back(L.getValue(1)); 3565 if (L->getValueType(0).isVector()) { 3566 SmallVector<SDValue, 16> Loads; 3567 Loads.push_back(L); 3568 unsigned size = L->getValueSizeInBits(0); 3569 while (size < LdOp->getValueSizeInBits(0)) { 3570 Loads.push_back(DAG.getUNDEF(L->getValueType(0))); 3571 size += L->getValueSizeInBits(0); 3572 } 3573 L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads); 3574 } 3575 } else { 3576 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 3577 LD->getPointerInfo().getWithOffset(Offset), isVolatile, 3578 isNonTemporal, isInvariant, MinAlign(Align, Increment), 3579 AAInfo); 3580 LdChain.push_back(L.getValue(1)); 3581 } 3582 3583 LdOps.push_back(L); 3584 3585 3586 LdWidth -= NewVTWidth; 3587 } 3588 3589 // Build the vector from the load operations. 3590 unsigned End = LdOps.size(); 3591 if (!LdOps[0].getValueType().isVector()) 3592 // All the loads are scalar loads. 3593 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End); 3594 3595 // If the load contains vectors, build the vector using concat vector. 3596 // All of the vectors used to load are power-of-2, and the scalar loads can be 3597 // combined to make a power-of-2 vector. 3598 SmallVector<SDValue, 16> ConcatOps(End); 3599 int i = End - 1; 3600 int Idx = End; 3601 EVT LdTy = LdOps[i].getValueType(); 3602 // First, combine the scalar loads to a vector. 3603 if (!LdTy.isVector()) { 3604 for (--i; i >= 0; --i) { 3605 LdTy = LdOps[i].getValueType(); 3606 if (LdTy.isVector()) 3607 break; 3608 } 3609 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End); 3610 } 3611 ConcatOps[--Idx] = LdOps[i]; 3612 for (--i; i >= 0; --i) { 3613 EVT NewLdTy = LdOps[i].getValueType(); 3614 if (NewLdTy != LdTy) { 3615 // Create a larger vector. 3616 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy, 3617 makeArrayRef(&ConcatOps[Idx], End - Idx)); 3618 Idx = End - 1; 3619 LdTy = NewLdTy; 3620 } 3621 ConcatOps[--Idx] = LdOps[i]; 3622 } 3623 3624 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx)) 3625 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 3626 makeArrayRef(&ConcatOps[Idx], End - Idx)); 3627 3628 // We need to fill the rest with undefs to build the vector. 3629 unsigned NumOps = WidenWidth / LdTy.getSizeInBits(); 3630 SmallVector<SDValue, 16> WidenOps(NumOps); 3631 SDValue UndefVal = DAG.getUNDEF(LdTy); 3632 { 3633 unsigned i = 0; 3634 for (; i != End-Idx; ++i) 3635 WidenOps[i] = ConcatOps[Idx+i]; 3636 for (; i != NumOps; ++i) 3637 WidenOps[i] = UndefVal; 3638 } 3639 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps); 3640 } 3641 3642 SDValue 3643 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain, 3644 LoadSDNode *LD, 3645 ISD::LoadExtType ExtType) { 3646 // For extension loads, it may not be more efficient to chop up the vector 3647 // and then extend it. Instead, we unroll the load and build a new vector. 3648 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 3649 EVT LdVT = LD->getMemoryVT(); 3650 SDLoc dl(LD); 3651 assert(LdVT.isVector() && WidenVT.isVector()); 3652 3653 // Load information 3654 SDValue Chain = LD->getChain(); 3655 SDValue BasePtr = LD->getBasePtr(); 3656 unsigned Align = LD->getAlignment(); 3657 bool isVolatile = LD->isVolatile(); 3658 bool isNonTemporal = LD->isNonTemporal(); 3659 bool isInvariant = LD->isInvariant(); 3660 AAMDNodes AAInfo = LD->getAAInfo(); 3661 3662 EVT EltVT = WidenVT.getVectorElementType(); 3663 EVT LdEltVT = LdVT.getVectorElementType(); 3664 unsigned NumElts = LdVT.getVectorNumElements(); 3665 3666 // Load each element and widen. 3667 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3668 SmallVector<SDValue, 16> Ops(WidenNumElts); 3669 unsigned Increment = LdEltVT.getSizeInBits() / 8; 3670 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, 3671 LD->getPointerInfo(), 3672 LdEltVT, isVolatile, isNonTemporal, isInvariant, 3673 Align, AAInfo); 3674 LdChain.push_back(Ops[0].getValue(1)); 3675 unsigned i = 0, Offset = Increment; 3676 for (i=1; i < NumElts; ++i, Offset += Increment) { 3677 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 3678 BasePtr, 3679 DAG.getConstant(Offset, dl, 3680 BasePtr.getValueType())); 3681 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, 3682 LD->getPointerInfo().getWithOffset(Offset), LdEltVT, 3683 isVolatile, isNonTemporal, isInvariant, Align, 3684 AAInfo); 3685 LdChain.push_back(Ops[i].getValue(1)); 3686 } 3687 3688 // Fill the rest with undefs. 3689 SDValue UndefVal = DAG.getUNDEF(EltVT); 3690 for (; i != WidenNumElts; ++i) 3691 Ops[i] = UndefVal; 3692 3693 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, Ops); 3694 } 3695 3696 3697 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, 3698 StoreSDNode *ST) { 3699 // The strategy assumes that we can efficiently store power-of-two widths. 3700 // The routine chops the vector into the largest vector stores with the same 3701 // element type or scalar stores. 3702 SDValue Chain = ST->getChain(); 3703 SDValue BasePtr = ST->getBasePtr(); 3704 unsigned Align = ST->getAlignment(); 3705 bool isVolatile = ST->isVolatile(); 3706 bool isNonTemporal = ST->isNonTemporal(); 3707 AAMDNodes AAInfo = ST->getAAInfo(); 3708 SDValue ValOp = GetWidenedVector(ST->getValue()); 3709 SDLoc dl(ST); 3710 3711 EVT StVT = ST->getMemoryVT(); 3712 unsigned StWidth = StVT.getSizeInBits(); 3713 EVT ValVT = ValOp.getValueType(); 3714 unsigned ValWidth = ValVT.getSizeInBits(); 3715 EVT ValEltVT = ValVT.getVectorElementType(); 3716 unsigned ValEltWidth = ValEltVT.getSizeInBits(); 3717 assert(StVT.getVectorElementType() == ValEltVT); 3718 3719 int Idx = 0; // current index to store 3720 unsigned Offset = 0; // offset from base to store 3721 while (StWidth != 0) { 3722 // Find the largest vector type we can store with. 3723 EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT); 3724 unsigned NewVTWidth = NewVT.getSizeInBits(); 3725 unsigned Increment = NewVTWidth / 8; 3726 if (NewVT.isVector()) { 3727 unsigned NumVTElts = NewVT.getVectorNumElements(); 3728 do { 3729 SDValue EOp = DAG.getNode( 3730 ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp, 3731 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3732 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 3733 ST->getPointerInfo().getWithOffset(Offset), 3734 isVolatile, isNonTemporal, 3735 MinAlign(Align, Offset), AAInfo)); 3736 StWidth -= NewVTWidth; 3737 Offset += Increment; 3738 Idx += NumVTElts; 3739 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 3740 DAG.getConstant(Increment, dl, 3741 BasePtr.getValueType())); 3742 } while (StWidth != 0 && StWidth >= NewVTWidth); 3743 } else { 3744 // Cast the vector to the scalar type we can store. 3745 unsigned NumElts = ValWidth / NewVTWidth; 3746 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 3747 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp); 3748 // Readjust index position based on new vector type. 3749 Idx = Idx * ValEltWidth / NewVTWidth; 3750 do { 3751 SDValue EOp = DAG.getNode( 3752 ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp, 3753 DAG.getConstant(Idx++, dl, 3754 TLI.getVectorIdxTy(DAG.getDataLayout()))); 3755 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 3756 ST->getPointerInfo().getWithOffset(Offset), 3757 isVolatile, isNonTemporal, 3758 MinAlign(Align, Offset), AAInfo)); 3759 StWidth -= NewVTWidth; 3760 Offset += Increment; 3761 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 3762 DAG.getConstant(Increment, dl, 3763 BasePtr.getValueType())); 3764 } while (StWidth != 0 && StWidth >= NewVTWidth); 3765 // Restore index back to be relative to the original widen element type. 3766 Idx = Idx * NewVTWidth / ValEltWidth; 3767 } 3768 } 3769 } 3770 3771 void 3772 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain, 3773 StoreSDNode *ST) { 3774 // For extension loads, it may not be more efficient to truncate the vector 3775 // and then store it. Instead, we extract each element and then store it. 3776 SDValue Chain = ST->getChain(); 3777 SDValue BasePtr = ST->getBasePtr(); 3778 unsigned Align = ST->getAlignment(); 3779 bool isVolatile = ST->isVolatile(); 3780 bool isNonTemporal = ST->isNonTemporal(); 3781 AAMDNodes AAInfo = ST->getAAInfo(); 3782 SDValue ValOp = GetWidenedVector(ST->getValue()); 3783 SDLoc dl(ST); 3784 3785 EVT StVT = ST->getMemoryVT(); 3786 EVT ValVT = ValOp.getValueType(); 3787 3788 // It must be true that the wide vector type is bigger than where we need to 3789 // store. 3790 assert(StVT.isVector() && ValOp.getValueType().isVector()); 3791 assert(StVT.bitsLT(ValOp.getValueType())); 3792 3793 // For truncating stores, we can not play the tricks of chopping legal vector 3794 // types and bitcast it to the right type. Instead, we unroll the store. 3795 EVT StEltVT = StVT.getVectorElementType(); 3796 EVT ValEltVT = ValVT.getVectorElementType(); 3797 unsigned Increment = ValEltVT.getSizeInBits() / 8; 3798 unsigned NumElts = StVT.getVectorNumElements(); 3799 SDValue EOp = DAG.getNode( 3800 ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 3801 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3802 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, 3803 ST->getPointerInfo(), StEltVT, 3804 isVolatile, isNonTemporal, Align, 3805 AAInfo)); 3806 unsigned Offset = Increment; 3807 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 3808 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 3809 BasePtr, 3810 DAG.getConstant(Offset, dl, 3811 BasePtr.getValueType())); 3812 SDValue EOp = DAG.getNode( 3813 ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 3814 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3815 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, 3816 ST->getPointerInfo().getWithOffset(Offset), 3817 StEltVT, isVolatile, isNonTemporal, 3818 MinAlign(Align, Offset), AAInfo)); 3819 } 3820 } 3821 3822 /// Modifies a vector input (widen or narrows) to a vector of NVT. The 3823 /// input vector must have the same element type as NVT. 3824 /// FillWithZeroes specifies that the vector should be widened with zeroes. 3825 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT, 3826 bool FillWithZeroes) { 3827 // Note that InOp might have been widened so it might already have 3828 // the right width or it might need be narrowed. 3829 EVT InVT = InOp.getValueType(); 3830 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 3831 "input and widen element type must match"); 3832 SDLoc dl(InOp); 3833 3834 // Check if InOp already has the right width. 3835 if (InVT == NVT) 3836 return InOp; 3837 3838 unsigned InNumElts = InVT.getVectorNumElements(); 3839 unsigned WidenNumElts = NVT.getVectorNumElements(); 3840 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 3841 unsigned NumConcat = WidenNumElts / InNumElts; 3842 SmallVector<SDValue, 16> Ops(NumConcat); 3843 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) : 3844 DAG.getUNDEF(InVT); 3845 Ops[0] = InOp; 3846 for (unsigned i = 1; i != NumConcat; ++i) 3847 Ops[i] = FillVal; 3848 3849 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops); 3850 } 3851 3852 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 3853 return DAG.getNode( 3854 ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 3855 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3856 3857 // Fall back to extract and build. 3858 SmallVector<SDValue, 16> Ops(WidenNumElts); 3859 EVT EltVT = NVT.getVectorElementType(); 3860 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 3861 unsigned Idx; 3862 for (Idx = 0; Idx < MinNumElts; ++Idx) 3863 Ops[Idx] = DAG.getNode( 3864 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 3865 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3866 3867 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) : 3868 DAG.getUNDEF(EltVT); 3869 for ( ; Idx < WidenNumElts; ++Idx) 3870 Ops[Idx] = FillVal; 3871 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, Ops); 3872 } 3873