1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file performs vector type splitting and scalarization for LegalizeTypes. 11 // Scalarization is the act of changing a computation in an illegal one-element 12 // vector type to be a computation in its scalar element type. For example, 13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which 15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16 // types. 17 // Splitting is the act of changing a computation in an invalid vector type to 18 // be a computation in two vectors of half the size. For example, implementing 19 // <128 x f32> operations in terms of two <64 x f32> operations. 20 // 21 //===----------------------------------------------------------------------===// 22 23 #include "LegalizeTypes.h" 24 #include "llvm/Target/TargetData.h" 25 #include "llvm/Support/ErrorHandling.h" 26 #include "llvm/Support/raw_ostream.h" 27 using namespace llvm; 28 29 //===----------------------------------------------------------------------===// 30 // Result Vector Scalarization: <1 x ty> -> ty. 31 //===----------------------------------------------------------------------===// 32 33 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 34 DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; 35 N->dump(&DAG); 36 dbgs() << "\n"); 37 SDValue R = SDValue(); 38 39 switch (N->getOpcode()) { 40 default: 41 #ifndef NDEBUG 42 dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 43 N->dump(&DAG); 44 dbgs() << "\n"; 45 #endif 46 report_fatal_error("Do not know how to scalarize the result of this " 47 "operator!\n"); 48 49 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; 50 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; 51 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; 52 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 53 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 54 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; 55 case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break; 56 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 57 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 58 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 59 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 60 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 61 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break; 62 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 63 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 64 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 65 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 66 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 67 case ISD::ANY_EXTEND: 68 case ISD::CTLZ: 69 case ISD::CTPOP: 70 case ISD::CTTZ: 71 case ISD::FABS: 72 case ISD::FCEIL: 73 case ISD::FCOS: 74 case ISD::FEXP: 75 case ISD::FEXP2: 76 case ISD::FFLOOR: 77 case ISD::FLOG: 78 case ISD::FLOG10: 79 case ISD::FLOG2: 80 case ISD::FNEARBYINT: 81 case ISD::FNEG: 82 case ISD::FP_EXTEND: 83 case ISD::FP_TO_SINT: 84 case ISD::FP_TO_UINT: 85 case ISD::FRINT: 86 case ISD::FSIN: 87 case ISD::FSQRT: 88 case ISD::FTRUNC: 89 case ISD::SIGN_EXTEND: 90 case ISD::SINT_TO_FP: 91 case ISD::TRUNCATE: 92 case ISD::UINT_TO_FP: 93 case ISD::ZERO_EXTEND: 94 R = ScalarizeVecRes_UnaryOp(N); 95 break; 96 97 case ISD::ADD: 98 case ISD::AND: 99 case ISD::FADD: 100 case ISD::FDIV: 101 case ISD::FMUL: 102 case ISD::FPOW: 103 case ISD::FREM: 104 case ISD::FSUB: 105 case ISD::MUL: 106 case ISD::OR: 107 case ISD::SDIV: 108 case ISD::SREM: 109 case ISD::SUB: 110 case ISD::UDIV: 111 case ISD::UREM: 112 case ISD::XOR: 113 case ISD::SHL: 114 case ISD::SRA: 115 case ISD::SRL: 116 R = ScalarizeVecRes_BinOp(N); 117 break; 118 case ISD::FMA: 119 R = ScalarizeVecRes_TernaryOp(N); 120 break; 121 } 122 123 // If R is null, the sub-method took care of registering the result. 124 if (R.getNode()) 125 SetScalarizedVector(SDValue(N, ResNo), R); 126 } 127 128 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 129 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 130 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 131 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 132 LHS.getValueType(), LHS, RHS); 133 } 134 135 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { 136 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 137 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 138 SDValue Op2 = GetScalarizedVector(N->getOperand(2)); 139 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 140 Op0.getValueType(), Op0, Op1, Op2); 141 } 142 143 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, 144 unsigned ResNo) { 145 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 146 return GetScalarizedVector(Op); 147 } 148 149 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { 150 EVT NewVT = N->getValueType(0).getVectorElementType(); 151 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), 152 NewVT, N->getOperand(0)); 153 } 154 155 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { 156 EVT EltVT = N->getValueType(0).getVectorElementType(); 157 SDValue InOp = N->getOperand(0); 158 // The BUILD_VECTOR operands may be of wider element types and 159 // we may need to truncate them back to the requested return type. 160 if (EltVT.isInteger()) 161 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 162 return InOp; 163 } 164 165 SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 166 EVT NewVT = N->getValueType(0).getVectorElementType(); 167 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 168 return DAG.getConvertRndSat(NewVT, N->getDebugLoc(), 169 Op0, DAG.getValueType(NewVT), 170 DAG.getValueType(Op0.getValueType()), 171 N->getOperand(3), 172 N->getOperand(4), 173 cast<CvtRndSatSDNode>(N)->getCvtCode()); 174 } 175 176 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 177 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 178 N->getValueType(0).getVectorElementType(), 179 N->getOperand(0), N->getOperand(1)); 180 } 181 182 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { 183 EVT NewVT = N->getValueType(0).getVectorElementType(); 184 SDValue Op = GetScalarizedVector(N->getOperand(0)); 185 return DAG.getNode(ISD::FP_ROUND, N->getDebugLoc(), 186 NewVT, Op, N->getOperand(1)); 187 } 188 189 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 190 SDValue Op = GetScalarizedVector(N->getOperand(0)); 191 return DAG.getNode(ISD::FPOWI, N->getDebugLoc(), 192 Op.getValueType(), Op, N->getOperand(1)); 193 } 194 195 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 196 // The value to insert may have a wider type than the vector element type, 197 // so be sure to truncate it to the element type if necessary. 198 SDValue Op = N->getOperand(1); 199 EVT EltVT = N->getValueType(0).getVectorElementType(); 200 if (Op.getValueType() != EltVT) 201 // FIXME: Can this happen for floating point types? 202 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op); 203 return Op; 204 } 205 206 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 207 assert(N->isUnindexed() && "Indexed vector load?"); 208 209 SDValue Result = DAG.getLoad(ISD::UNINDEXED, 210 N->getExtensionType(), 211 N->getValueType(0).getVectorElementType(), 212 N->getDebugLoc(), 213 N->getChain(), N->getBasePtr(), 214 DAG.getUNDEF(N->getBasePtr().getValueType()), 215 N->getPointerInfo(), 216 N->getMemoryVT().getVectorElementType(), 217 N->isVolatile(), N->isNonTemporal(), 218 N->isInvariant(), N->getOriginalAlignment()); 219 220 // Legalized the chain result - switch anything that used the old chain to 221 // use the new one. 222 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 223 return Result; 224 } 225 226 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 227 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 228 EVT DestVT = N->getValueType(0).getVectorElementType(); 229 SDValue Op = GetScalarizedVector(N->getOperand(0)); 230 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op); 231 } 232 233 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 234 EVT EltVT = N->getValueType(0).getVectorElementType(); 235 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 236 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 237 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), EltVT, 238 LHS, DAG.getValueType(ExtVT)); 239 } 240 241 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 242 // If the operand is wider than the vector element type then it is implicitly 243 // truncated. Make that explicit here. 244 EVT EltVT = N->getValueType(0).getVectorElementType(); 245 SDValue InOp = N->getOperand(0); 246 if (InOp.getValueType() != EltVT) 247 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 248 return InOp; 249 } 250 251 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { 252 SDValue Cond = GetScalarizedVector(N->getOperand(0)); 253 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 254 TargetLowering::BooleanContent ScalarBool = TLI.getBooleanContents(false); 255 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true); 256 if (ScalarBool != VecBool) { 257 EVT CondVT = Cond.getValueType(); 258 switch (ScalarBool) { 259 case TargetLowering::UndefinedBooleanContent: 260 break; 261 case TargetLowering::ZeroOrOneBooleanContent: 262 assert(VecBool == TargetLowering::UndefinedBooleanContent || 263 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent); 264 // Vector read from all ones, scalar expects a single 1 so mask. 265 Cond = DAG.getNode(ISD::AND, N->getDebugLoc(), CondVT, 266 Cond, DAG.getConstant(1, CondVT)); 267 break; 268 case TargetLowering::ZeroOrNegativeOneBooleanContent: 269 assert(VecBool == TargetLowering::UndefinedBooleanContent || 270 VecBool == TargetLowering::ZeroOrOneBooleanContent); 271 // Vector reads from a one, scalar from all ones so sign extend. 272 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), CondVT, 273 Cond, DAG.getValueType(MVT::i1)); 274 break; 275 } 276 } 277 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 278 LHS.getValueType(), Cond, LHS, 279 GetScalarizedVector(N->getOperand(2))); 280 } 281 282 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 283 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 284 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 285 LHS.getValueType(), N->getOperand(0), LHS, 286 GetScalarizedVector(N->getOperand(2))); 287 } 288 289 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 290 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 291 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(), 292 N->getOperand(0), N->getOperand(1), 293 LHS, GetScalarizedVector(N->getOperand(3)), 294 N->getOperand(4)); 295 } 296 297 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 298 assert(N->getValueType(0).isVector() == 299 N->getOperand(0).getValueType().isVector() && 300 "Scalar/Vector type mismatch"); 301 302 if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N); 303 304 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 305 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 306 DebugLoc DL = N->getDebugLoc(); 307 308 // Turn it into a scalar SETCC. 309 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 310 } 311 312 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 313 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 314 } 315 316 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 317 // Figure out if the scalar is the LHS or RHS and return it. 318 SDValue Arg = N->getOperand(2).getOperand(0); 319 if (Arg.getOpcode() == ISD::UNDEF) 320 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 321 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 322 return GetScalarizedVector(N->getOperand(Op)); 323 } 324 325 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 326 assert(N->getValueType(0).isVector() && 327 N->getOperand(0).getValueType().isVector() && 328 "Operand types must be vectors"); 329 330 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 331 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 332 EVT NVT = N->getValueType(0).getVectorElementType(); 333 DebugLoc DL = N->getDebugLoc(); 334 335 // Turn it into a scalar SETCC. 336 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 337 N->getOperand(2)); 338 // Vectors may have a different boolean contents to scalars. Promote the 339 // value appropriately. 340 ISD::NodeType ExtendCode = 341 TargetLowering::getExtendForContent(TLI.getBooleanContents(true)); 342 return DAG.getNode(ExtendCode, DL, NVT, Res); 343 } 344 345 346 //===----------------------------------------------------------------------===// 347 // Operand Vector Scalarization <1 x ty> -> ty. 348 //===----------------------------------------------------------------------===// 349 350 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 351 DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; 352 N->dump(&DAG); 353 dbgs() << "\n"); 354 SDValue Res = SDValue(); 355 356 if (Res.getNode() == 0) { 357 switch (N->getOpcode()) { 358 default: 359 #ifndef NDEBUG 360 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 361 N->dump(&DAG); 362 dbgs() << "\n"; 363 #endif 364 llvm_unreachable("Do not know how to scalarize this operator's operand!"); 365 case ISD::BITCAST: 366 Res = ScalarizeVecOp_BITCAST(N); 367 break; 368 case ISD::CONCAT_VECTORS: 369 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 370 break; 371 case ISD::EXTRACT_VECTOR_ELT: 372 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 373 break; 374 case ISD::STORE: 375 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 376 break; 377 } 378 } 379 380 // If the result is null, the sub-method took care of registering results etc. 381 if (!Res.getNode()) return false; 382 383 // If the result is N, the sub-method updated N in place. Tell the legalizer 384 // core about this. 385 if (Res.getNode() == N) 386 return true; 387 388 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 389 "Invalid operand expansion"); 390 391 ReplaceValueWith(SDValue(N, 0), Res); 392 return false; 393 } 394 395 /// ScalarizeVecOp_BITCAST - If the value to convert is a vector that needs 396 /// to be scalarized, it must be <1 x ty>. Convert the element instead. 397 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { 398 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 399 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), 400 N->getValueType(0), Elt); 401 } 402 403 /// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 404 /// use a BUILD_VECTOR instead. 405 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 406 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 407 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 408 Ops[i] = GetScalarizedVector(N->getOperand(i)); 409 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 410 &Ops[0], Ops.size()); 411 } 412 413 /// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 414 /// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 415 /// index. 416 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 417 SDValue Res = GetScalarizedVector(N->getOperand(0)); 418 if (Res.getValueType() != N->getValueType(0)) 419 Res = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), 420 Res); 421 return Res; 422 } 423 424 /// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 425 /// scalarized, it must be <1 x ty>. Just store the element. 426 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 427 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 428 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 429 DebugLoc dl = N->getDebugLoc(); 430 431 if (N->isTruncatingStore()) 432 return DAG.getTruncStore(N->getChain(), dl, 433 GetScalarizedVector(N->getOperand(1)), 434 N->getBasePtr(), N->getPointerInfo(), 435 N->getMemoryVT().getVectorElementType(), 436 N->isVolatile(), N->isNonTemporal(), 437 N->getAlignment()); 438 439 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 440 N->getBasePtr(), N->getPointerInfo(), 441 N->isVolatile(), N->isNonTemporal(), 442 N->getOriginalAlignment()); 443 } 444 445 446 //===----------------------------------------------------------------------===// 447 // Result Vector Splitting 448 //===----------------------------------------------------------------------===// 449 450 /// SplitVectorResult - This method is called when the specified result of the 451 /// specified node is found to need vector splitting. At this point, the node 452 /// may also have invalid operands or may have other results that need 453 /// legalization, we just know that (at least) one result needs vector 454 /// splitting. 455 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 456 DEBUG(dbgs() << "Split node result: "; 457 N->dump(&DAG); 458 dbgs() << "\n"); 459 SDValue Lo, Hi; 460 461 // See if the target wants to custom expand this node. 462 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 463 return; 464 465 switch (N->getOpcode()) { 466 default: 467 #ifndef NDEBUG 468 dbgs() << "SplitVectorResult #" << ResNo << ": "; 469 N->dump(&DAG); 470 dbgs() << "\n"; 471 #endif 472 report_fatal_error("Do not know how to split the result of this " 473 "operator!\n"); 474 475 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 476 case ISD::VSELECT: 477 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 478 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 479 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 480 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break; 481 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 482 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 483 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 484 case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 485 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 486 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 487 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 488 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 489 case ISD::LOAD: 490 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 491 break; 492 case ISD::SETCC: 493 SplitVecRes_SETCC(N, Lo, Hi); 494 break; 495 case ISD::VECTOR_SHUFFLE: 496 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 497 break; 498 499 case ISD::ANY_EXTEND: 500 case ISD::CONVERT_RNDSAT: 501 case ISD::CTLZ: 502 case ISD::CTTZ: 503 case ISD::CTLZ_ZERO_UNDEF: 504 case ISD::CTTZ_ZERO_UNDEF: 505 case ISD::CTPOP: 506 case ISD::FABS: 507 case ISD::FCEIL: 508 case ISD::FCOS: 509 case ISD::FEXP: 510 case ISD::FEXP2: 511 case ISD::FFLOOR: 512 case ISD::FLOG: 513 case ISD::FLOG10: 514 case ISD::FLOG2: 515 case ISD::FNEARBYINT: 516 case ISD::FNEG: 517 case ISD::FP_EXTEND: 518 case ISD::FP_ROUND: 519 case ISD::FP_TO_SINT: 520 case ISD::FP_TO_UINT: 521 case ISD::FRINT: 522 case ISD::FSIN: 523 case ISD::FSQRT: 524 case ISD::FTRUNC: 525 case ISD::SIGN_EXTEND: 526 case ISD::SINT_TO_FP: 527 case ISD::TRUNCATE: 528 case ISD::UINT_TO_FP: 529 case ISD::ZERO_EXTEND: 530 SplitVecRes_UnaryOp(N, Lo, Hi); 531 break; 532 533 case ISD::ADD: 534 case ISD::SUB: 535 case ISD::MUL: 536 case ISD::FADD: 537 case ISD::FSUB: 538 case ISD::FMUL: 539 case ISD::SDIV: 540 case ISD::UDIV: 541 case ISD::FDIV: 542 case ISD::FPOW: 543 case ISD::AND: 544 case ISD::OR: 545 case ISD::XOR: 546 case ISD::SHL: 547 case ISD::SRA: 548 case ISD::SRL: 549 case ISD::UREM: 550 case ISD::SREM: 551 case ISD::FREM: 552 SplitVecRes_BinOp(N, Lo, Hi); 553 break; 554 case ISD::FMA: 555 SplitVecRes_TernaryOp(N, Lo, Hi); 556 break; 557 } 558 559 // If Lo/Hi is null, the sub-method took care of registering results etc. 560 if (Lo.getNode()) 561 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 562 } 563 564 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 565 SDValue &Hi) { 566 SDValue LHSLo, LHSHi; 567 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 568 SDValue RHSLo, RHSHi; 569 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 570 DebugLoc dl = N->getDebugLoc(); 571 572 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 573 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 574 } 575 576 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, 577 SDValue &Hi) { 578 SDValue Op0Lo, Op0Hi; 579 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 580 SDValue Op1Lo, Op1Hi; 581 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 582 SDValue Op2Lo, Op2Hi; 583 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi); 584 DebugLoc dl = N->getDebugLoc(); 585 586 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), 587 Op0Lo, Op1Lo, Op2Lo); 588 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), 589 Op0Hi, Op1Hi, Op2Hi); 590 } 591 592 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, 593 SDValue &Hi) { 594 // We know the result is a vector. The input may be either a vector or a 595 // scalar value. 596 EVT LoVT, HiVT; 597 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 598 DebugLoc dl = N->getDebugLoc(); 599 600 SDValue InOp = N->getOperand(0); 601 EVT InVT = InOp.getValueType(); 602 603 // Handle some special cases efficiently. 604 switch (getTypeAction(InVT)) { 605 case TargetLowering::TypeLegal: 606 case TargetLowering::TypePromoteInteger: 607 case TargetLowering::TypeSoftenFloat: 608 case TargetLowering::TypeScalarizeVector: 609 case TargetLowering::TypeWidenVector: 610 break; 611 case TargetLowering::TypeExpandInteger: 612 case TargetLowering::TypeExpandFloat: 613 // A scalar to vector conversion, where the scalar needs expansion. 614 // If the vector is being split in two then we can just convert the 615 // expanded pieces. 616 if (LoVT == HiVT) { 617 GetExpandedOp(InOp, Lo, Hi); 618 if (TLI.isBigEndian()) 619 std::swap(Lo, Hi); 620 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 621 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 622 return; 623 } 624 break; 625 case TargetLowering::TypeSplitVector: 626 // If the input is a vector that needs to be split, convert each split 627 // piece of the input now. 628 GetSplitVector(InOp, Lo, Hi); 629 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 630 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 631 return; 632 } 633 634 // In the general case, convert the input to an integer and split it by hand. 635 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 636 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 637 if (TLI.isBigEndian()) 638 std::swap(LoIntVT, HiIntVT); 639 640 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 641 642 if (TLI.isBigEndian()) 643 std::swap(Lo, Hi); 644 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 645 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 646 } 647 648 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 649 SDValue &Hi) { 650 EVT LoVT, HiVT; 651 DebugLoc dl = N->getDebugLoc(); 652 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 653 unsigned LoNumElts = LoVT.getVectorNumElements(); 654 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 655 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size()); 656 657 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 658 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size()); 659 } 660 661 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 662 SDValue &Hi) { 663 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 664 DebugLoc dl = N->getDebugLoc(); 665 unsigned NumSubvectors = N->getNumOperands() / 2; 666 if (NumSubvectors == 1) { 667 Lo = N->getOperand(0); 668 Hi = N->getOperand(1); 669 return; 670 } 671 672 EVT LoVT, HiVT; 673 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 674 675 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 676 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size()); 677 678 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 679 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size()); 680 } 681 682 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 683 SDValue &Hi) { 684 SDValue Vec = N->getOperand(0); 685 SDValue Idx = N->getOperand(1); 686 DebugLoc dl = N->getDebugLoc(); 687 688 EVT LoVT, HiVT; 689 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 690 691 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 692 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 693 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, 694 DAG.getIntPtrConstant(IdxVal + LoVT.getVectorNumElements())); 695 } 696 697 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 698 SDValue &Hi) { 699 DebugLoc dl = N->getDebugLoc(); 700 GetSplitVector(N->getOperand(0), Lo, Hi); 701 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 702 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 703 } 704 705 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 706 SDValue &Hi) { 707 SDValue LHSLo, LHSHi; 708 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 709 DebugLoc dl = N->getDebugLoc(); 710 711 EVT LoVT, HiVT; 712 GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT(), LoVT, HiVT); 713 714 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 715 DAG.getValueType(LoVT)); 716 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 717 DAG.getValueType(HiVT)); 718 } 719 720 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 721 SDValue &Hi) { 722 SDValue Vec = N->getOperand(0); 723 SDValue Elt = N->getOperand(1); 724 SDValue Idx = N->getOperand(2); 725 DebugLoc dl = N->getDebugLoc(); 726 GetSplitVector(Vec, Lo, Hi); 727 728 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 729 unsigned IdxVal = CIdx->getZExtValue(); 730 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 731 if (IdxVal < LoNumElts) 732 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 733 Lo.getValueType(), Lo, Elt, Idx); 734 else 735 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 736 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 737 return; 738 } 739 740 // Spill the vector to the stack. 741 EVT VecVT = Vec.getValueType(); 742 EVT EltVT = VecVT.getVectorElementType(); 743 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 744 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 745 MachinePointerInfo(), false, false, 0); 746 747 // Store the new element. This may be larger than the vector element type, 748 // so use a truncating store. 749 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 750 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 751 unsigned Alignment = 752 TLI.getTargetData()->getPrefTypeAlignment(VecType); 753 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT, 754 false, false, 0); 755 756 // Load the Lo part from the stack slot. 757 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 758 false, false, false, 0); 759 760 // Increment the pointer to the other part. 761 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 762 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 763 DAG.getIntPtrConstant(IncrementSize)); 764 765 // Load the Hi part from the stack slot. 766 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 767 false, false, false, MinAlign(Alignment, IncrementSize)); 768 } 769 770 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 771 SDValue &Hi) { 772 EVT LoVT, HiVT; 773 DebugLoc dl = N->getDebugLoc(); 774 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 775 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 776 Hi = DAG.getUNDEF(HiVT); 777 } 778 779 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 780 SDValue &Hi) { 781 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 782 EVT LoVT, HiVT; 783 DebugLoc dl = LD->getDebugLoc(); 784 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 785 786 ISD::LoadExtType ExtType = LD->getExtensionType(); 787 SDValue Ch = LD->getChain(); 788 SDValue Ptr = LD->getBasePtr(); 789 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 790 EVT MemoryVT = LD->getMemoryVT(); 791 unsigned Alignment = LD->getOriginalAlignment(); 792 bool isVolatile = LD->isVolatile(); 793 bool isNonTemporal = LD->isNonTemporal(); 794 bool isInvariant = LD->isInvariant(); 795 796 EVT LoMemVT, HiMemVT; 797 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 798 799 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset, 800 LD->getPointerInfo(), LoMemVT, isVolatile, isNonTemporal, 801 isInvariant, Alignment); 802 803 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 804 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 805 DAG.getIntPtrConstant(IncrementSize)); 806 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, 807 LD->getPointerInfo().getWithOffset(IncrementSize), 808 HiMemVT, isVolatile, isNonTemporal, isInvariant, Alignment); 809 810 // Build a factor node to remember that this load is independent of the 811 // other one. 812 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 813 Hi.getValue(1)); 814 815 // Legalized the chain result - switch anything that used the old chain to 816 // use the new one. 817 ReplaceValueWith(SDValue(LD, 1), Ch); 818 } 819 820 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 821 assert(N->getValueType(0).isVector() && 822 N->getOperand(0).getValueType().isVector() && 823 "Operand types must be vectors"); 824 825 EVT LoVT, HiVT; 826 DebugLoc DL = N->getDebugLoc(); 827 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 828 829 // Split the input. 830 EVT InVT = N->getOperand(0).getValueType(); 831 SDValue LL, LH, RL, RH; 832 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 833 LoVT.getVectorNumElements()); 834 LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 835 DAG.getIntPtrConstant(0)); 836 LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 837 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 838 839 RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 840 DAG.getIntPtrConstant(0)); 841 RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 842 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 843 844 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 845 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 846 } 847 848 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 849 SDValue &Hi) { 850 // Get the dest types - they may not match the input types, e.g. int_to_fp. 851 EVT LoVT, HiVT; 852 DebugLoc dl = N->getDebugLoc(); 853 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 854 855 // If the input also splits, handle it directly for a compile time speedup. 856 // Otherwise split it by hand. 857 EVT InVT = N->getOperand(0).getValueType(); 858 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) { 859 GetSplitVector(N->getOperand(0), Lo, Hi); 860 } else { 861 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 862 LoVT.getVectorNumElements()); 863 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 864 DAG.getIntPtrConstant(0)); 865 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 866 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 867 } 868 869 if (N->getOpcode() == ISD::FP_ROUND) { 870 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1)); 871 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1)); 872 } else if (N->getOpcode() == ISD::CONVERT_RNDSAT) { 873 SDValue DTyOpLo = DAG.getValueType(LoVT); 874 SDValue DTyOpHi = DAG.getValueType(HiVT); 875 SDValue STyOpLo = DAG.getValueType(Lo.getValueType()); 876 SDValue STyOpHi = DAG.getValueType(Hi.getValueType()); 877 SDValue RndOp = N->getOperand(3); 878 SDValue SatOp = N->getOperand(4); 879 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 880 Lo = DAG.getConvertRndSat(LoVT, dl, Lo, DTyOpLo, STyOpLo, RndOp, SatOp, 881 CvtCode); 882 Hi = DAG.getConvertRndSat(HiVT, dl, Hi, DTyOpHi, STyOpHi, RndOp, SatOp, 883 CvtCode); 884 } else { 885 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 886 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 887 } 888 } 889 890 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 891 SDValue &Lo, SDValue &Hi) { 892 // The low and high parts of the original input give four input vectors. 893 SDValue Inputs[4]; 894 DebugLoc dl = N->getDebugLoc(); 895 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 896 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 897 EVT NewVT = Inputs[0].getValueType(); 898 unsigned NewElts = NewVT.getVectorNumElements(); 899 900 // If Lo or Hi uses elements from at most two of the four input vectors, then 901 // express it as a vector shuffle of those two inputs. Otherwise extract the 902 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 903 SmallVector<int, 16> Ops; 904 for (unsigned High = 0; High < 2; ++High) { 905 SDValue &Output = High ? Hi : Lo; 906 907 // Build a shuffle mask for the output, discovering on the fly which 908 // input vectors to use as shuffle operands (recorded in InputUsed). 909 // If building a suitable shuffle vector proves too hard, then bail 910 // out with useBuildVector set. 911 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 912 unsigned FirstMaskIdx = High * NewElts; 913 bool useBuildVector = false; 914 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 915 // The mask element. This indexes into the input. 916 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 917 918 // The input vector this mask element indexes into. 919 unsigned Input = (unsigned)Idx / NewElts; 920 921 if (Input >= array_lengthof(Inputs)) { 922 // The mask element does not index into any input vector. 923 Ops.push_back(-1); 924 continue; 925 } 926 927 // Turn the index into an offset from the start of the input vector. 928 Idx -= Input * NewElts; 929 930 // Find or create a shuffle vector operand to hold this input. 931 unsigned OpNo; 932 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 933 if (InputUsed[OpNo] == Input) { 934 // This input vector is already an operand. 935 break; 936 } else if (InputUsed[OpNo] == -1U) { 937 // Create a new operand for this input vector. 938 InputUsed[OpNo] = Input; 939 break; 940 } 941 } 942 943 if (OpNo >= array_lengthof(InputUsed)) { 944 // More than two input vectors used! Give up on trying to create a 945 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 946 useBuildVector = true; 947 break; 948 } 949 950 // Add the mask index for the new shuffle vector. 951 Ops.push_back(Idx + OpNo * NewElts); 952 } 953 954 if (useBuildVector) { 955 EVT EltVT = NewVT.getVectorElementType(); 956 SmallVector<SDValue, 16> SVOps; 957 958 // Extract the input elements by hand. 959 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 960 // The mask element. This indexes into the input. 961 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 962 963 // The input vector this mask element indexes into. 964 unsigned Input = (unsigned)Idx / NewElts; 965 966 if (Input >= array_lengthof(Inputs)) { 967 // The mask element is "undef" or indexes off the end of the input. 968 SVOps.push_back(DAG.getUNDEF(EltVT)); 969 continue; 970 } 971 972 // Turn the index into an offset from the start of the input vector. 973 Idx -= Input * NewElts; 974 975 // Extract the vector element by hand. 976 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 977 Inputs[Input], DAG.getIntPtrConstant(Idx))); 978 } 979 980 // Construct the Lo/Hi output using a BUILD_VECTOR. 981 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size()); 982 } else if (InputUsed[0] == -1U) { 983 // No input vectors were used! The result is undefined. 984 Output = DAG.getUNDEF(NewVT); 985 } else { 986 SDValue Op0 = Inputs[InputUsed[0]]; 987 // If only one input was used, use an undefined vector for the other. 988 SDValue Op1 = InputUsed[1] == -1U ? 989 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 990 // At least one input vector was used. Create a new shuffle vector. 991 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 992 } 993 994 Ops.clear(); 995 } 996 } 997 998 999 //===----------------------------------------------------------------------===// 1000 // Operand Vector Splitting 1001 //===----------------------------------------------------------------------===// 1002 1003 /// SplitVectorOperand - This method is called when the specified operand of the 1004 /// specified node is found to need vector splitting. At this point, all of the 1005 /// result types of the node are known to be legal, but other operands of the 1006 /// node may need legalization as well as the specified one. 1007 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 1008 DEBUG(dbgs() << "Split node operand: "; 1009 N->dump(&DAG); 1010 dbgs() << "\n"); 1011 SDValue Res = SDValue(); 1012 1013 if (Res.getNode() == 0) { 1014 switch (N->getOpcode()) { 1015 default: 1016 #ifndef NDEBUG 1017 dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 1018 N->dump(&DAG); 1019 dbgs() << "\n"; 1020 #endif 1021 report_fatal_error("Do not know how to split this operator's " 1022 "operand!\n"); 1023 1024 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; 1025 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; 1026 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 1027 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 1028 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; 1029 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; 1030 case ISD::STORE: 1031 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 1032 break; 1033 1034 case ISD::CTTZ: 1035 case ISD::CTLZ: 1036 case ISD::CTPOP: 1037 case ISD::FP_EXTEND: 1038 case ISD::FP_TO_SINT: 1039 case ISD::FP_TO_UINT: 1040 case ISD::SINT_TO_FP: 1041 case ISD::UINT_TO_FP: 1042 case ISD::FTRUNC: 1043 case ISD::TRUNCATE: 1044 case ISD::SIGN_EXTEND: 1045 case ISD::ZERO_EXTEND: 1046 case ISD::ANY_EXTEND: 1047 Res = SplitVecOp_UnaryOp(N); 1048 break; 1049 } 1050 } 1051 1052 // If the result is null, the sub-method took care of registering results etc. 1053 if (!Res.getNode()) return false; 1054 1055 // If the result is N, the sub-method updated N in place. Tell the legalizer 1056 // core about this. 1057 if (Res.getNode() == N) 1058 return true; 1059 1060 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1061 "Invalid operand expansion"); 1062 1063 ReplaceValueWith(SDValue(N, 0), Res); 1064 return false; 1065 } 1066 1067 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 1068 // The result has a legal vector type, but the input needs splitting. 1069 EVT ResVT = N->getValueType(0); 1070 SDValue Lo, Hi; 1071 DebugLoc dl = N->getDebugLoc(); 1072 GetSplitVector(N->getOperand(0), Lo, Hi); 1073 EVT InVT = Lo.getValueType(); 1074 1075 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1076 InVT.getVectorNumElements()); 1077 1078 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 1079 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 1080 1081 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 1082 } 1083 1084 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { 1085 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will 1086 // end up being split all the way down to individual components. Convert the 1087 // split pieces into integers and reassemble. 1088 SDValue Lo, Hi; 1089 GetSplitVector(N->getOperand(0), Lo, Hi); 1090 Lo = BitConvertToInteger(Lo); 1091 Hi = BitConvertToInteger(Hi); 1092 1093 if (TLI.isBigEndian()) 1094 std::swap(Lo, Hi); 1095 1096 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), N->getValueType(0), 1097 JoinIntegers(Lo, Hi)); 1098 } 1099 1100 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1101 // We know that the extracted result type is legal. 1102 EVT SubVT = N->getValueType(0); 1103 SDValue Idx = N->getOperand(1); 1104 DebugLoc dl = N->getDebugLoc(); 1105 SDValue Lo, Hi; 1106 GetSplitVector(N->getOperand(0), Lo, Hi); 1107 1108 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1109 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1110 1111 if (IdxVal < LoElts) { 1112 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1113 "Extracted subvector crosses vector split!"); 1114 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1115 } else { 1116 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1117 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1118 } 1119 } 1120 1121 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1122 SDValue Vec = N->getOperand(0); 1123 SDValue Idx = N->getOperand(1); 1124 EVT VecVT = Vec.getValueType(); 1125 1126 if (isa<ConstantSDNode>(Idx)) { 1127 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1128 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1129 1130 SDValue Lo, Hi; 1131 GetSplitVector(Vec, Lo, Hi); 1132 1133 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1134 1135 if (IdxVal < LoElts) 1136 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0); 1137 return SDValue(DAG.UpdateNodeOperands(N, Hi, 1138 DAG.getConstant(IdxVal - LoElts, 1139 Idx.getValueType())), 0); 1140 } 1141 1142 // Store the vector to the stack. 1143 EVT EltVT = VecVT.getVectorElementType(); 1144 DebugLoc dl = N->getDebugLoc(); 1145 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1146 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1147 MachinePointerInfo(), false, false, 0); 1148 1149 // Load back the required element. 1150 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1151 return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 1152 MachinePointerInfo(), EltVT, false, false, 0); 1153 } 1154 1155 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1156 assert(N->isUnindexed() && "Indexed store of vector?"); 1157 assert(OpNo == 1 && "Can only split the stored value"); 1158 DebugLoc DL = N->getDebugLoc(); 1159 1160 bool isTruncating = N->isTruncatingStore(); 1161 SDValue Ch = N->getChain(); 1162 SDValue Ptr = N->getBasePtr(); 1163 EVT MemoryVT = N->getMemoryVT(); 1164 unsigned Alignment = N->getOriginalAlignment(); 1165 bool isVol = N->isVolatile(); 1166 bool isNT = N->isNonTemporal(); 1167 SDValue Lo, Hi; 1168 GetSplitVector(N->getOperand(1), Lo, Hi); 1169 1170 EVT LoMemVT, HiMemVT; 1171 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 1172 1173 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1174 1175 if (isTruncating) 1176 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1177 LoMemVT, isVol, isNT, Alignment); 1178 else 1179 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1180 isVol, isNT, Alignment); 1181 1182 // Increment the pointer to the other half. 1183 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 1184 DAG.getIntPtrConstant(IncrementSize)); 1185 1186 if (isTruncating) 1187 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, 1188 N->getPointerInfo().getWithOffset(IncrementSize), 1189 HiMemVT, isVol, isNT, Alignment); 1190 else 1191 Hi = DAG.getStore(Ch, DL, Hi, Ptr, 1192 N->getPointerInfo().getWithOffset(IncrementSize), 1193 isVol, isNT, Alignment); 1194 1195 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 1196 } 1197 1198 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { 1199 DebugLoc DL = N->getDebugLoc(); 1200 1201 // The input operands all must have the same type, and we know the result the 1202 // result type is valid. Convert this to a buildvector which extracts all the 1203 // input elements. 1204 // TODO: If the input elements are power-two vectors, we could convert this to 1205 // a new CONCAT_VECTORS node with elements that are half-wide. 1206 SmallVector<SDValue, 32> Elts; 1207 EVT EltVT = N->getValueType(0).getVectorElementType(); 1208 for (unsigned op = 0, e = N->getNumOperands(); op != e; ++op) { 1209 SDValue Op = N->getOperand(op); 1210 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements(); 1211 i != e; ++i) { 1212 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, 1213 Op, DAG.getIntPtrConstant(i))); 1214 1215 } 1216 } 1217 1218 return DAG.getNode(ISD::BUILD_VECTOR, DL, N->getValueType(0), 1219 &Elts[0], Elts.size()); 1220 } 1221 1222 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { 1223 assert(N->getValueType(0).isVector() && 1224 N->getOperand(0).getValueType().isVector() && 1225 "Operand types must be vectors"); 1226 // The result has a legal vector type, but the input needs splitting. 1227 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; 1228 DebugLoc DL = N->getDebugLoc(); 1229 GetSplitVector(N->getOperand(0), Lo0, Hi0); 1230 GetSplitVector(N->getOperand(1), Lo1, Hi1); 1231 unsigned PartElements = Lo0.getValueType().getVectorNumElements(); 1232 EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements); 1233 EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements); 1234 1235 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2)); 1236 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2)); 1237 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes); 1238 return PromoteTargetBoolean(Con, N->getValueType(0)); 1239 } 1240 1241 1242 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { 1243 // The result has a legal vector type, but the input needs splitting. 1244 EVT ResVT = N->getValueType(0); 1245 SDValue Lo, Hi; 1246 DebugLoc DL = N->getDebugLoc(); 1247 GetSplitVector(N->getOperand(0), Lo, Hi); 1248 EVT InVT = Lo.getValueType(); 1249 1250 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1251 InVT.getVectorNumElements()); 1252 1253 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); 1254 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); 1255 1256 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); 1257 } 1258 1259 1260 1261 //===----------------------------------------------------------------------===// 1262 // Result Vector Widening 1263 //===----------------------------------------------------------------------===// 1264 1265 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1266 DEBUG(dbgs() << "Widen node result " << ResNo << ": "; 1267 N->dump(&DAG); 1268 dbgs() << "\n"); 1269 1270 // See if the target wants to custom widen this node. 1271 if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 1272 return; 1273 1274 SDValue Res = SDValue(); 1275 switch (N->getOpcode()) { 1276 default: 1277 #ifndef NDEBUG 1278 dbgs() << "WidenVectorResult #" << ResNo << ": "; 1279 N->dump(&DAG); 1280 dbgs() << "\n"; 1281 #endif 1282 llvm_unreachable("Do not know how to widen the result of this operator!"); 1283 1284 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; 1285 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; 1286 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1287 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1288 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1289 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1290 case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break; 1291 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1292 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1293 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1294 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 1295 case ISD::VSELECT: 1296 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1297 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1298 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break; 1299 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1300 case ISD::VECTOR_SHUFFLE: 1301 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 1302 break; 1303 case ISD::ADD: 1304 case ISD::AND: 1305 case ISD::BSWAP: 1306 case ISD::FADD: 1307 case ISD::FCOPYSIGN: 1308 case ISD::FDIV: 1309 case ISD::FMUL: 1310 case ISD::FPOW: 1311 case ISD::FREM: 1312 case ISD::FSUB: 1313 case ISD::MUL: 1314 case ISD::MULHS: 1315 case ISD::MULHU: 1316 case ISD::OR: 1317 case ISD::SDIV: 1318 case ISD::SREM: 1319 case ISD::UDIV: 1320 case ISD::UREM: 1321 case ISD::SUB: 1322 case ISD::XOR: 1323 Res = WidenVecRes_Binary(N); 1324 break; 1325 1326 case ISD::FPOWI: 1327 Res = WidenVecRes_POWI(N); 1328 break; 1329 1330 case ISD::SHL: 1331 case ISD::SRA: 1332 case ISD::SRL: 1333 Res = WidenVecRes_Shift(N); 1334 break; 1335 1336 case ISD::ANY_EXTEND: 1337 case ISD::FP_EXTEND: 1338 case ISD::FP_ROUND: 1339 case ISD::FP_TO_SINT: 1340 case ISD::FP_TO_UINT: 1341 case ISD::SIGN_EXTEND: 1342 case ISD::SINT_TO_FP: 1343 case ISD::TRUNCATE: 1344 case ISD::UINT_TO_FP: 1345 case ISD::ZERO_EXTEND: 1346 Res = WidenVecRes_Convert(N); 1347 break; 1348 1349 case ISD::CTLZ: 1350 case ISD::CTPOP: 1351 case ISD::CTTZ: 1352 case ISD::FABS: 1353 case ISD::FCEIL: 1354 case ISD::FCOS: 1355 case ISD::FEXP: 1356 case ISD::FEXP2: 1357 case ISD::FFLOOR: 1358 case ISD::FLOG: 1359 case ISD::FLOG10: 1360 case ISD::FLOG2: 1361 case ISD::FNEARBYINT: 1362 case ISD::FNEG: 1363 case ISD::FRINT: 1364 case ISD::FSIN: 1365 case ISD::FSQRT: 1366 case ISD::FTRUNC: 1367 Res = WidenVecRes_Unary(N); 1368 break; 1369 case ISD::FMA: 1370 Res = WidenVecRes_Ternary(N); 1371 break; 1372 } 1373 1374 // If Res is null, the sub-method took care of registering the result. 1375 if (Res.getNode()) 1376 SetWidenedVector(SDValue(N, ResNo), Res); 1377 } 1378 1379 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { 1380 // Ternary op widening. 1381 DebugLoc dl = N->getDebugLoc(); 1382 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1383 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1384 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1385 SDValue InOp3 = GetWidenedVector(N->getOperand(2)); 1386 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3); 1387 } 1388 1389 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1390 // Binary op widening. 1391 unsigned Opcode = N->getOpcode(); 1392 DebugLoc dl = N->getDebugLoc(); 1393 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1394 EVT WidenEltVT = WidenVT.getVectorElementType(); 1395 EVT VT = WidenVT; 1396 unsigned NumElts = VT.getVectorNumElements(); 1397 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 1398 NumElts = NumElts / 2; 1399 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1400 } 1401 1402 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) { 1403 // Operation doesn't trap so just widen as normal. 1404 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1405 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1406 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2); 1407 } 1408 1409 // No legal vector version so unroll the vector operation and then widen. 1410 if (NumElts == 1) 1411 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 1412 1413 // Since the operation can trap, apply operation on the original vector. 1414 EVT MaxVT = VT; 1415 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1416 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1417 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 1418 1419 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 1420 unsigned ConcatEnd = 0; // Current ConcatOps index. 1421 int Idx = 0; // Current Idx into input vectors. 1422 1423 // NumElts := greatest legal vector size (at most WidenVT) 1424 // while (orig. vector has unhandled elements) { 1425 // take munches of size NumElts from the beginning and add to ConcatOps 1426 // NumElts := next smaller supported vector size or 1 1427 // } 1428 while (CurNumElts != 0) { 1429 while (CurNumElts >= NumElts) { 1430 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1, 1431 DAG.getIntPtrConstant(Idx)); 1432 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2, 1433 DAG.getIntPtrConstant(Idx)); 1434 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2); 1435 Idx += NumElts; 1436 CurNumElts -= NumElts; 1437 } 1438 do { 1439 NumElts = NumElts / 2; 1440 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1441 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 1442 1443 if (NumElts == 1) { 1444 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 1445 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1446 InOp1, DAG.getIntPtrConstant(Idx)); 1447 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1448 InOp2, DAG.getIntPtrConstant(Idx)); 1449 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT, 1450 EOp1, EOp2); 1451 } 1452 CurNumElts = 0; 1453 } 1454 } 1455 1456 // Check to see if we have a single operation with the widen type. 1457 if (ConcatEnd == 1) { 1458 VT = ConcatOps[0].getValueType(); 1459 if (VT == WidenVT) 1460 return ConcatOps[0]; 1461 } 1462 1463 // while (Some element of ConcatOps is not of type MaxVT) { 1464 // From the end of ConcatOps, collect elements of the same type and put 1465 // them into an op of the next larger supported type 1466 // } 1467 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { 1468 Idx = ConcatEnd - 1; 1469 VT = ConcatOps[Idx--].getValueType(); 1470 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) 1471 Idx--; 1472 1473 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1; 1474 EVT NextVT; 1475 do { 1476 NextSize *= 2; 1477 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize); 1478 } while (!TLI.isTypeLegal(NextVT)); 1479 1480 if (!VT.isVector()) { 1481 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT 1482 SDValue VecOp = DAG.getUNDEF(NextVT); 1483 unsigned NumToInsert = ConcatEnd - Idx - 1; 1484 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) { 1485 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, 1486 ConcatOps[OpIdx], DAG.getIntPtrConstant(i)); 1487 } 1488 ConcatOps[Idx+1] = VecOp; 1489 ConcatEnd = Idx + 2; 1490 } else { 1491 // Vector type, create a CONCAT_VECTORS of type NextVT 1492 SDValue undefVec = DAG.getUNDEF(VT); 1493 unsigned OpsToConcat = NextSize/VT.getVectorNumElements(); 1494 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat); 1495 unsigned RealVals = ConcatEnd - Idx - 1; 1496 unsigned SubConcatEnd = 0; 1497 unsigned SubConcatIdx = Idx + 1; 1498 while (SubConcatEnd < RealVals) 1499 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx]; 1500 while (SubConcatEnd < OpsToConcat) 1501 SubConcatOps[SubConcatEnd++] = undefVec; 1502 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1503 NextVT, &SubConcatOps[0], 1504 OpsToConcat); 1505 ConcatEnd = SubConcatIdx + 1; 1506 } 1507 } 1508 1509 // Check to see if we have a single operation with the widen type. 1510 if (ConcatEnd == 1) { 1511 VT = ConcatOps[0].getValueType(); 1512 if (VT == WidenVT) 1513 return ConcatOps[0]; 1514 } 1515 1516 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT 1517 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements(); 1518 if (NumOps != ConcatEnd ) { 1519 SDValue UndefVal = DAG.getUNDEF(MaxVT); 1520 for (unsigned j = ConcatEnd; j < NumOps; ++j) 1521 ConcatOps[j] = UndefVal; 1522 } 1523 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &ConcatOps[0], NumOps); 1524 } 1525 1526 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1527 SDValue InOp = N->getOperand(0); 1528 DebugLoc DL = N->getDebugLoc(); 1529 1530 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1531 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1532 1533 EVT InVT = InOp.getValueType(); 1534 EVT InEltVT = InVT.getVectorElementType(); 1535 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1536 1537 unsigned Opcode = N->getOpcode(); 1538 unsigned InVTNumElts = InVT.getVectorNumElements(); 1539 1540 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 1541 InOp = GetWidenedVector(N->getOperand(0)); 1542 InVT = InOp.getValueType(); 1543 InVTNumElts = InVT.getVectorNumElements(); 1544 if (InVTNumElts == WidenNumElts) { 1545 if (N->getNumOperands() == 1) 1546 return DAG.getNode(Opcode, DL, WidenVT, InOp); 1547 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1)); 1548 } 1549 } 1550 1551 if (TLI.isTypeLegal(InWidenVT)) { 1552 // Because the result and the input are different vector types, widening 1553 // the result could create a legal type but widening the input might make 1554 // it an illegal type that might lead to repeatedly splitting the input 1555 // and then widening it. To avoid this, we widen the input only if 1556 // it results in a legal type. 1557 if (WidenNumElts % InVTNumElts == 0) { 1558 // Widen the input and call convert on the widened input vector. 1559 unsigned NumConcat = WidenNumElts/InVTNumElts; 1560 SmallVector<SDValue, 16> Ops(NumConcat); 1561 Ops[0] = InOp; 1562 SDValue UndefVal = DAG.getUNDEF(InVT); 1563 for (unsigned i = 1; i != NumConcat; ++i) 1564 Ops[i] = UndefVal; 1565 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, 1566 &Ops[0], NumConcat); 1567 if (N->getNumOperands() == 1) 1568 return DAG.getNode(Opcode, DL, WidenVT, InVec); 1569 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1)); 1570 } 1571 1572 if (InVTNumElts % WidenNumElts == 0) { 1573 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, 1574 InOp, DAG.getIntPtrConstant(0)); 1575 // Extract the input and convert the shorten input vector. 1576 if (N->getNumOperands() == 1) 1577 return DAG.getNode(Opcode, DL, WidenVT, InVal); 1578 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1)); 1579 } 1580 } 1581 1582 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1583 SmallVector<SDValue, 16> Ops(WidenNumElts); 1584 EVT EltVT = WidenVT.getVectorElementType(); 1585 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1586 unsigned i; 1587 for (i=0; i < MinElts; ++i) { 1588 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 1589 DAG.getIntPtrConstant(i)); 1590 if (N->getNumOperands() == 1) 1591 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val); 1592 else 1593 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1)); 1594 } 1595 1596 SDValue UndefVal = DAG.getUNDEF(EltVT); 1597 for (; i < WidenNumElts; ++i) 1598 Ops[i] = UndefVal; 1599 1600 return DAG.getNode(ISD::BUILD_VECTOR, DL, WidenVT, &Ops[0], WidenNumElts); 1601 } 1602 1603 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) { 1604 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1605 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1606 SDValue ShOp = N->getOperand(1); 1607 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1608 } 1609 1610 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 1611 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1612 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1613 SDValue ShOp = N->getOperand(1); 1614 1615 EVT ShVT = ShOp.getValueType(); 1616 if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) { 1617 ShOp = GetWidenedVector(ShOp); 1618 ShVT = ShOp.getValueType(); 1619 } 1620 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), 1621 ShVT.getVectorElementType(), 1622 WidenVT.getVectorNumElements()); 1623 if (ShVT != ShWidenVT) 1624 ShOp = ModifyToType(ShOp, ShWidenVT); 1625 1626 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1627 } 1628 1629 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 1630 // Unary op widening. 1631 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1632 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1633 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp); 1634 } 1635 1636 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 1637 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1638 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 1639 cast<VTSDNode>(N->getOperand(1))->getVT() 1640 .getVectorElementType(), 1641 WidenVT.getVectorNumElements()); 1642 SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 1643 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 1644 WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 1645 } 1646 1647 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { 1648 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); 1649 return GetWidenedVector(WidenVec); 1650 } 1651 1652 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { 1653 SDValue InOp = N->getOperand(0); 1654 EVT InVT = InOp.getValueType(); 1655 EVT VT = N->getValueType(0); 1656 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1657 DebugLoc dl = N->getDebugLoc(); 1658 1659 switch (getTypeAction(InVT)) { 1660 case TargetLowering::TypeLegal: 1661 break; 1662 case TargetLowering::TypePromoteInteger: 1663 // If the incoming type is a vector that is being promoted, then 1664 // we know that the elements are arranged differently and that we 1665 // must perform the conversion using a stack slot. 1666 if (InVT.isVector()) 1667 break; 1668 1669 // If the InOp is promoted to the same size, convert it. Otherwise, 1670 // fall out of the switch and widen the promoted input. 1671 InOp = GetPromotedInteger(InOp); 1672 InVT = InOp.getValueType(); 1673 if (WidenVT.bitsEq(InVT)) 1674 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 1675 break; 1676 case TargetLowering::TypeSoftenFloat: 1677 case TargetLowering::TypeExpandInteger: 1678 case TargetLowering::TypeExpandFloat: 1679 case TargetLowering::TypeScalarizeVector: 1680 case TargetLowering::TypeSplitVector: 1681 break; 1682 case TargetLowering::TypeWidenVector: 1683 // If the InOp is widened to the same size, convert it. Otherwise, fall 1684 // out of the switch and widen the widened input. 1685 InOp = GetWidenedVector(InOp); 1686 InVT = InOp.getValueType(); 1687 if (WidenVT.bitsEq(InVT)) 1688 // The input widens to the same size. Convert to the widen value. 1689 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 1690 break; 1691 } 1692 1693 unsigned WidenSize = WidenVT.getSizeInBits(); 1694 unsigned InSize = InVT.getSizeInBits(); 1695 // x86mmx is not an acceptable vector element type, so don't try. 1696 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) { 1697 // Determine new input vector type. The new input vector type will use 1698 // the same element type (if its a vector) or use the input type as a 1699 // vector. It is the same size as the type to widen to. 1700 EVT NewInVT; 1701 unsigned NewNumElts = WidenSize / InSize; 1702 if (InVT.isVector()) { 1703 EVT InEltVT = InVT.getVectorElementType(); 1704 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, 1705 WidenSize / InEltVT.getSizeInBits()); 1706 } else { 1707 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 1708 } 1709 1710 if (TLI.isTypeLegal(NewInVT)) { 1711 // Because the result and the input are different vector types, widening 1712 // the result could create a legal type but widening the input might make 1713 // it an illegal type that might lead to repeatedly splitting the input 1714 // and then widening it. To avoid this, we widen the input only if 1715 // it results in a legal type. 1716 SmallVector<SDValue, 16> Ops(NewNumElts); 1717 SDValue UndefVal = DAG.getUNDEF(InVT); 1718 Ops[0] = InOp; 1719 for (unsigned i = 1; i < NewNumElts; ++i) 1720 Ops[i] = UndefVal; 1721 1722 SDValue NewVec; 1723 if (InVT.isVector()) 1724 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1725 NewInVT, &Ops[0], NewNumElts); 1726 else 1727 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 1728 NewInVT, &Ops[0], NewNumElts); 1729 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec); 1730 } 1731 } 1732 1733 return CreateStackStoreLoad(InOp, WidenVT); 1734 } 1735 1736 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 1737 DebugLoc dl = N->getDebugLoc(); 1738 // Build a vector with undefined for the new nodes. 1739 EVT VT = N->getValueType(0); 1740 EVT EltVT = VT.getVectorElementType(); 1741 unsigned NumElts = VT.getVectorNumElements(); 1742 1743 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1744 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1745 1746 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 1747 NewOps.reserve(WidenNumElts); 1748 for (unsigned i = NumElts; i < WidenNumElts; ++i) 1749 NewOps.push_back(DAG.getUNDEF(EltVT)); 1750 1751 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size()); 1752 } 1753 1754 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 1755 EVT InVT = N->getOperand(0).getValueType(); 1756 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1757 DebugLoc dl = N->getDebugLoc(); 1758 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1759 unsigned NumInElts = InVT.getVectorNumElements(); 1760 unsigned NumOperands = N->getNumOperands(); 1761 1762 bool InputWidened = false; // Indicates we need to widen the input. 1763 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) { 1764 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 1765 // Add undef vectors to widen to correct length. 1766 unsigned NumConcat = WidenVT.getVectorNumElements() / 1767 InVT.getVectorNumElements(); 1768 SDValue UndefVal = DAG.getUNDEF(InVT); 1769 SmallVector<SDValue, 16> Ops(NumConcat); 1770 for (unsigned i=0; i < NumOperands; ++i) 1771 Ops[i] = N->getOperand(i); 1772 for (unsigned i = NumOperands; i != NumConcat; ++i) 1773 Ops[i] = UndefVal; 1774 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat); 1775 } 1776 } else { 1777 InputWidened = true; 1778 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 1779 // The inputs and the result are widen to the same value. 1780 unsigned i; 1781 for (i=1; i < NumOperands; ++i) 1782 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 1783 break; 1784 1785 if (i == NumOperands) 1786 // Everything but the first operand is an UNDEF so just return the 1787 // widened first operand. 1788 return GetWidenedVector(N->getOperand(0)); 1789 1790 if (NumOperands == 2) { 1791 // Replace concat of two operands with a shuffle. 1792 SmallVector<int, 16> MaskOps(WidenNumElts, -1); 1793 for (unsigned i = 0; i < NumInElts; ++i) { 1794 MaskOps[i] = i; 1795 MaskOps[i + NumInElts] = i + WidenNumElts; 1796 } 1797 return DAG.getVectorShuffle(WidenVT, dl, 1798 GetWidenedVector(N->getOperand(0)), 1799 GetWidenedVector(N->getOperand(1)), 1800 &MaskOps[0]); 1801 } 1802 } 1803 } 1804 1805 // Fall back to use extracts and build vector. 1806 EVT EltVT = WidenVT.getVectorElementType(); 1807 SmallVector<SDValue, 16> Ops(WidenNumElts); 1808 unsigned Idx = 0; 1809 for (unsigned i=0; i < NumOperands; ++i) { 1810 SDValue InOp = N->getOperand(i); 1811 if (InputWidened) 1812 InOp = GetWidenedVector(InOp); 1813 for (unsigned j=0; j < NumInElts; ++j) 1814 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1815 DAG.getIntPtrConstant(j)); 1816 } 1817 SDValue UndefVal = DAG.getUNDEF(EltVT); 1818 for (; Idx < WidenNumElts; ++Idx) 1819 Ops[Idx] = UndefVal; 1820 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1821 } 1822 1823 SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 1824 DebugLoc dl = N->getDebugLoc(); 1825 SDValue InOp = N->getOperand(0); 1826 SDValue RndOp = N->getOperand(3); 1827 SDValue SatOp = N->getOperand(4); 1828 1829 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1830 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1831 1832 EVT InVT = InOp.getValueType(); 1833 EVT InEltVT = InVT.getVectorElementType(); 1834 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1835 1836 SDValue DTyOp = DAG.getValueType(WidenVT); 1837 SDValue STyOp = DAG.getValueType(InWidenVT); 1838 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1839 1840 unsigned InVTNumElts = InVT.getVectorNumElements(); 1841 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 1842 InOp = GetWidenedVector(InOp); 1843 InVT = InOp.getValueType(); 1844 InVTNumElts = InVT.getVectorNumElements(); 1845 if (InVTNumElts == WidenNumElts) 1846 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1847 SatOp, CvtCode); 1848 } 1849 1850 if (TLI.isTypeLegal(InWidenVT)) { 1851 // Because the result and the input are different vector types, widening 1852 // the result could create a legal type but widening the input might make 1853 // it an illegal type that might lead to repeatedly splitting the input 1854 // and then widening it. To avoid this, we widen the input only if 1855 // it results in a legal type. 1856 if (WidenNumElts % InVTNumElts == 0) { 1857 // Widen the input and call convert on the widened input vector. 1858 unsigned NumConcat = WidenNumElts/InVTNumElts; 1859 SmallVector<SDValue, 16> Ops(NumConcat); 1860 Ops[0] = InOp; 1861 SDValue UndefVal = DAG.getUNDEF(InVT); 1862 for (unsigned i = 1; i != NumConcat; ++i) 1863 Ops[i] = UndefVal; 1864 1865 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat); 1866 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1867 SatOp, CvtCode); 1868 } 1869 1870 if (InVTNumElts % WidenNumElts == 0) { 1871 // Extract the input and convert the shorten input vector. 1872 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 1873 DAG.getIntPtrConstant(0)); 1874 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1875 SatOp, CvtCode); 1876 } 1877 } 1878 1879 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1880 SmallVector<SDValue, 16> Ops(WidenNumElts); 1881 EVT EltVT = WidenVT.getVectorElementType(); 1882 DTyOp = DAG.getValueType(EltVT); 1883 STyOp = DAG.getValueType(InEltVT); 1884 1885 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1886 unsigned i; 1887 for (i=0; i < MinElts; ++i) { 1888 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1889 DAG.getIntPtrConstant(i)); 1890 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 1891 SatOp, CvtCode); 1892 } 1893 1894 SDValue UndefVal = DAG.getUNDEF(EltVT); 1895 for (; i < WidenNumElts; ++i) 1896 Ops[i] = UndefVal; 1897 1898 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1899 } 1900 1901 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 1902 EVT VT = N->getValueType(0); 1903 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1904 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1905 SDValue InOp = N->getOperand(0); 1906 SDValue Idx = N->getOperand(1); 1907 DebugLoc dl = N->getDebugLoc(); 1908 1909 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 1910 InOp = GetWidenedVector(InOp); 1911 1912 EVT InVT = InOp.getValueType(); 1913 1914 // Check if we can just return the input vector after widening. 1915 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1916 if (IdxVal == 0 && InVT == WidenVT) 1917 return InOp; 1918 1919 // Check if we can extract from the vector. 1920 unsigned InNumElts = InVT.getVectorNumElements(); 1921 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 1922 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 1923 1924 // We could try widening the input to the right length but for now, extract 1925 // the original elements, fill the rest with undefs and build a vector. 1926 SmallVector<SDValue, 16> Ops(WidenNumElts); 1927 EVT EltVT = VT.getVectorElementType(); 1928 unsigned NumElts = VT.getVectorNumElements(); 1929 unsigned i; 1930 for (i=0; i < NumElts; ++i) 1931 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1932 DAG.getIntPtrConstant(IdxVal+i)); 1933 1934 SDValue UndefVal = DAG.getUNDEF(EltVT); 1935 for (; i < WidenNumElts; ++i) 1936 Ops[i] = UndefVal; 1937 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1938 } 1939 1940 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 1941 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1942 return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(), 1943 InOp.getValueType(), InOp, 1944 N->getOperand(1), N->getOperand(2)); 1945 } 1946 1947 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 1948 LoadSDNode *LD = cast<LoadSDNode>(N); 1949 ISD::LoadExtType ExtType = LD->getExtensionType(); 1950 1951 SDValue Result; 1952 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 1953 if (ExtType != ISD::NON_EXTLOAD) 1954 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); 1955 else 1956 Result = GenWidenVectorLoads(LdChain, LD); 1957 1958 // If we generate a single load, we can use that for the chain. Otherwise, 1959 // build a factor node to remember the multiple loads are independent and 1960 // chain to that. 1961 SDValue NewChain; 1962 if (LdChain.size() == 1) 1963 NewChain = LdChain[0]; 1964 else 1965 NewChain = DAG.getNode(ISD::TokenFactor, LD->getDebugLoc(), MVT::Other, 1966 &LdChain[0], LdChain.size()); 1967 1968 // Modified the chain - switch anything that used the old chain to use 1969 // the new one. 1970 ReplaceValueWith(SDValue(N, 1), NewChain); 1971 1972 return Result; 1973 } 1974 1975 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 1976 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1977 return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(), 1978 WidenVT, N->getOperand(0)); 1979 } 1980 1981 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 1982 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1983 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1984 1985 SDValue Cond1 = N->getOperand(0); 1986 EVT CondVT = Cond1.getValueType(); 1987 if (CondVT.isVector()) { 1988 EVT CondEltVT = CondVT.getVectorElementType(); 1989 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), 1990 CondEltVT, WidenNumElts); 1991 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector) 1992 Cond1 = GetWidenedVector(Cond1); 1993 1994 if (Cond1.getValueType() != CondWidenVT) 1995 Cond1 = ModifyToType(Cond1, CondWidenVT); 1996 } 1997 1998 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 1999 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 2000 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 2001 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 2002 WidenVT, Cond1, InOp1, InOp2); 2003 } 2004 2005 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 2006 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 2007 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 2008 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 2009 InOp1.getValueType(), N->getOperand(0), 2010 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 2011 } 2012 2013 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { 2014 assert(N->getValueType(0).isVector() == 2015 N->getOperand(0).getValueType().isVector() && 2016 "Scalar/Vector type mismatch"); 2017 if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N); 2018 2019 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2020 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2021 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2022 return DAG.getNode(ISD::SETCC, N->getDebugLoc(), WidenVT, 2023 InOp1, InOp2, N->getOperand(2)); 2024 } 2025 2026 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 2027 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2028 return DAG.getUNDEF(WidenVT); 2029 } 2030 2031 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 2032 EVT VT = N->getValueType(0); 2033 DebugLoc dl = N->getDebugLoc(); 2034 2035 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2036 unsigned NumElts = VT.getVectorNumElements(); 2037 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2038 2039 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2040 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2041 2042 // Adjust mask based on new input vector length. 2043 SmallVector<int, 16> NewMask; 2044 for (unsigned i = 0; i != NumElts; ++i) { 2045 int Idx = N->getMaskElt(i); 2046 if (Idx < (int)NumElts) 2047 NewMask.push_back(Idx); 2048 else 2049 NewMask.push_back(Idx - NumElts + WidenNumElts); 2050 } 2051 for (unsigned i = NumElts; i != WidenNumElts; ++i) 2052 NewMask.push_back(-1); 2053 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 2054 } 2055 2056 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 2057 assert(N->getValueType(0).isVector() && 2058 N->getOperand(0).getValueType().isVector() && 2059 "Operands must be vectors"); 2060 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2061 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2062 2063 SDValue InOp1 = N->getOperand(0); 2064 EVT InVT = InOp1.getValueType(); 2065 assert(InVT.isVector() && "can not widen non vector type"); 2066 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), 2067 InVT.getVectorElementType(), WidenNumElts); 2068 InOp1 = GetWidenedVector(InOp1); 2069 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2070 2071 // Assume that the input and output will be widen appropriately. If not, 2072 // we will have to unroll it at some point. 2073 assert(InOp1.getValueType() == WidenInVT && 2074 InOp2.getValueType() == WidenInVT && 2075 "Input not widened to expected type!"); 2076 (void)WidenInVT; 2077 return DAG.getNode(ISD::SETCC, N->getDebugLoc(), 2078 WidenVT, InOp1, InOp2, N->getOperand(2)); 2079 } 2080 2081 2082 //===----------------------------------------------------------------------===// 2083 // Widen Vector Operand 2084 //===----------------------------------------------------------------------===// 2085 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) { 2086 DEBUG(dbgs() << "Widen node operand " << ResNo << ": "; 2087 N->dump(&DAG); 2088 dbgs() << "\n"); 2089 SDValue Res = SDValue(); 2090 2091 switch (N->getOpcode()) { 2092 default: 2093 #ifndef NDEBUG 2094 dbgs() << "WidenVectorOperand op #" << ResNo << ": "; 2095 N->dump(&DAG); 2096 dbgs() << "\n"; 2097 #endif 2098 llvm_unreachable("Do not know how to widen this operator's operand!"); 2099 2100 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break; 2101 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 2102 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 2103 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 2104 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 2105 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; 2106 2107 case ISD::FP_EXTEND: 2108 case ISD::FP_TO_SINT: 2109 case ISD::FP_TO_UINT: 2110 case ISD::SINT_TO_FP: 2111 case ISD::UINT_TO_FP: 2112 case ISD::TRUNCATE: 2113 case ISD::SIGN_EXTEND: 2114 case ISD::ZERO_EXTEND: 2115 case ISD::ANY_EXTEND: 2116 Res = WidenVecOp_Convert(N); 2117 break; 2118 } 2119 2120 // If Res is null, the sub-method took care of registering the result. 2121 if (!Res.getNode()) return false; 2122 2123 // If the result is N, the sub-method updated N in place. Tell the legalizer 2124 // core about this. 2125 if (Res.getNode() == N) 2126 return true; 2127 2128 2129 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2130 "Invalid operand expansion"); 2131 2132 ReplaceValueWith(SDValue(N, 0), Res); 2133 return false; 2134 } 2135 2136 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 2137 // Since the result is legal and the input is illegal, it is unlikely 2138 // that we can fix the input to a legal type so unroll the convert 2139 // into some scalar code and create a nasty build vector. 2140 EVT VT = N->getValueType(0); 2141 EVT EltVT = VT.getVectorElementType(); 2142 DebugLoc dl = N->getDebugLoc(); 2143 unsigned NumElts = VT.getVectorNumElements(); 2144 SDValue InOp = N->getOperand(0); 2145 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2146 InOp = GetWidenedVector(InOp); 2147 EVT InVT = InOp.getValueType(); 2148 EVT InEltVT = InVT.getVectorElementType(); 2149 2150 unsigned Opcode = N->getOpcode(); 2151 SmallVector<SDValue, 16> Ops(NumElts); 2152 for (unsigned i=0; i < NumElts; ++i) 2153 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 2154 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 2155 DAG.getIntPtrConstant(i))); 2156 2157 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 2158 } 2159 2160 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { 2161 EVT VT = N->getValueType(0); 2162 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2163 EVT InWidenVT = InOp.getValueType(); 2164 DebugLoc dl = N->getDebugLoc(); 2165 2166 // Check if we can convert between two legal vector types and extract. 2167 unsigned InWidenSize = InWidenVT.getSizeInBits(); 2168 unsigned Size = VT.getSizeInBits(); 2169 // x86mmx is not an acceptable vector element type, so don't try. 2170 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) { 2171 unsigned NewNumElts = InWidenSize / Size; 2172 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 2173 if (TLI.isTypeLegal(NewVT)) { 2174 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 2175 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 2176 DAG.getIntPtrConstant(0)); 2177 } 2178 } 2179 2180 return CreateStackStoreLoad(InOp, VT); 2181 } 2182 2183 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 2184 // If the input vector is not legal, it is likely that we will not find a 2185 // legal vector of the same size. Replace the concatenate vector with a 2186 // nasty build vector. 2187 EVT VT = N->getValueType(0); 2188 EVT EltVT = VT.getVectorElementType(); 2189 DebugLoc dl = N->getDebugLoc(); 2190 unsigned NumElts = VT.getVectorNumElements(); 2191 SmallVector<SDValue, 16> Ops(NumElts); 2192 2193 EVT InVT = N->getOperand(0).getValueType(); 2194 unsigned NumInElts = InVT.getVectorNumElements(); 2195 2196 unsigned Idx = 0; 2197 unsigned NumOperands = N->getNumOperands(); 2198 for (unsigned i=0; i < NumOperands; ++i) { 2199 SDValue InOp = N->getOperand(i); 2200 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2201 InOp = GetWidenedVector(InOp); 2202 for (unsigned j=0; j < NumInElts; ++j) 2203 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2204 DAG.getIntPtrConstant(j)); 2205 } 2206 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 2207 } 2208 2209 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 2210 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2211 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), 2212 N->getValueType(0), InOp, N->getOperand(1)); 2213 } 2214 2215 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 2216 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2217 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 2218 N->getValueType(0), InOp, N->getOperand(1)); 2219 } 2220 2221 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 2222 // We have to widen the value but we want only to store the original 2223 // vector type. 2224 StoreSDNode *ST = cast<StoreSDNode>(N); 2225 2226 SmallVector<SDValue, 16> StChain; 2227 if (ST->isTruncatingStore()) 2228 GenWidenVectorTruncStores(StChain, ST); 2229 else 2230 GenWidenVectorStores(StChain, ST); 2231 2232 if (StChain.size() == 1) 2233 return StChain[0]; 2234 else 2235 return DAG.getNode(ISD::TokenFactor, ST->getDebugLoc(), 2236 MVT::Other,&StChain[0],StChain.size()); 2237 } 2238 2239 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { 2240 SDValue InOp0 = GetWidenedVector(N->getOperand(0)); 2241 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 2242 DebugLoc dl = N->getDebugLoc(); 2243 2244 // WARNING: In this code we widen the compare instruction with garbage. 2245 // This garbage may contain denormal floats which may be slow. Is this a real 2246 // concern ? Should we zero the unused lanes if this is a float compare ? 2247 2248 // Get a new SETCC node to compare the newly widened operands. 2249 // Only some of the compared elements are legal. 2250 EVT SVT = TLI.getSetCCResultType(InOp0.getValueType()); 2251 SDValue WideSETCC = DAG.getNode(ISD::SETCC, N->getDebugLoc(), 2252 SVT, InOp0, InOp1, N->getOperand(2)); 2253 2254 // Extract the needed results from the result vector. 2255 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), 2256 SVT.getVectorElementType(), 2257 N->getValueType(0).getVectorNumElements()); 2258 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, 2259 ResVT, WideSETCC, DAG.getIntPtrConstant(0)); 2260 2261 return PromoteTargetBoolean(CC, N->getValueType(0)); 2262 } 2263 2264 2265 //===----------------------------------------------------------------------===// 2266 // Vector Widening Utilities 2267 //===----------------------------------------------------------------------===// 2268 2269 // Utility function to find the type to chop up a widen vector for load/store 2270 // TLI: Target lowering used to determine legal types. 2271 // Width: Width left need to load/store. 2272 // WidenVT: The widen vector type to load to/store from 2273 // Align: If 0, don't allow use of a wider type 2274 // WidenEx: If Align is not 0, the amount additional we can load/store from. 2275 2276 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI, 2277 unsigned Width, EVT WidenVT, 2278 unsigned Align = 0, unsigned WidenEx = 0) { 2279 EVT WidenEltVT = WidenVT.getVectorElementType(); 2280 unsigned WidenWidth = WidenVT.getSizeInBits(); 2281 unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); 2282 unsigned AlignInBits = Align*8; 2283 2284 // If we have one element to load/store, return it. 2285 EVT RetVT = WidenEltVT; 2286 if (Width == WidenEltWidth) 2287 return RetVT; 2288 2289 // See if there is larger legal integer than the element type to load/store 2290 unsigned VT; 2291 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE; 2292 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) { 2293 EVT MemVT((MVT::SimpleValueType) VT); 2294 unsigned MemVTWidth = MemVT.getSizeInBits(); 2295 if (MemVT.getSizeInBits() <= WidenEltWidth) 2296 break; 2297 if (TLI.isTypeLegal(MemVT) && (WidenWidth % MemVTWidth) == 0 && 2298 isPowerOf2_32(WidenWidth / MemVTWidth) && 2299 (MemVTWidth <= Width || 2300 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2301 RetVT = MemVT; 2302 break; 2303 } 2304 } 2305 2306 // See if there is a larger vector type to load/store that has the same vector 2307 // element type and is evenly divisible with the WidenVT. 2308 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE; 2309 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) { 2310 EVT MemVT = (MVT::SimpleValueType) VT; 2311 unsigned MemVTWidth = MemVT.getSizeInBits(); 2312 if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() && 2313 (WidenWidth % MemVTWidth) == 0 && 2314 isPowerOf2_32(WidenWidth / MemVTWidth) && 2315 (MemVTWidth <= Width || 2316 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2317 if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT) 2318 return MemVT; 2319 } 2320 } 2321 2322 return RetVT; 2323 } 2324 2325 // Builds a vector type from scalar loads 2326 // VecTy: Resulting Vector type 2327 // LDOps: Load operators to build a vector type 2328 // [Start,End) the list of loads to use. 2329 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, 2330 SmallVector<SDValue, 16>& LdOps, 2331 unsigned Start, unsigned End) { 2332 DebugLoc dl = LdOps[Start].getDebugLoc(); 2333 EVT LdTy = LdOps[Start].getValueType(); 2334 unsigned Width = VecTy.getSizeInBits(); 2335 unsigned NumElts = Width / LdTy.getSizeInBits(); 2336 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts); 2337 2338 unsigned Idx = 1; 2339 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]); 2340 2341 for (unsigned i = Start + 1; i != End; ++i) { 2342 EVT NewLdTy = LdOps[i].getValueType(); 2343 if (NewLdTy != LdTy) { 2344 NumElts = Width / NewLdTy.getSizeInBits(); 2345 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts); 2346 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp); 2347 // Readjust position and vector position based on new load type 2348 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); 2349 LdTy = NewLdTy; 2350 } 2351 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i], 2352 DAG.getIntPtrConstant(Idx++)); 2353 } 2354 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp); 2355 } 2356 2357 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16> &LdChain, 2358 LoadSDNode *LD) { 2359 // The strategy assumes that we can efficiently load powers of two widths. 2360 // The routines chops the vector into the largest vector loads with the same 2361 // element type or scalar loads and then recombines it to the widen vector 2362 // type. 2363 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2364 unsigned WidenWidth = WidenVT.getSizeInBits(); 2365 EVT LdVT = LD->getMemoryVT(); 2366 DebugLoc dl = LD->getDebugLoc(); 2367 assert(LdVT.isVector() && WidenVT.isVector()); 2368 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 2369 2370 // Load information 2371 SDValue Chain = LD->getChain(); 2372 SDValue BasePtr = LD->getBasePtr(); 2373 unsigned Align = LD->getAlignment(); 2374 bool isVolatile = LD->isVolatile(); 2375 bool isNonTemporal = LD->isNonTemporal(); 2376 bool isInvariant = LD->isInvariant(); 2377 2378 int LdWidth = LdVT.getSizeInBits(); 2379 int WidthDiff = WidenWidth - LdWidth; // Difference 2380 unsigned LdAlign = (isVolatile) ? 0 : Align; // Allow wider loads 2381 2382 // Find the vector type that can load from. 2383 EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2384 int NewVTWidth = NewVT.getSizeInBits(); 2385 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(), 2386 isVolatile, isNonTemporal, isInvariant, Align); 2387 LdChain.push_back(LdOp.getValue(1)); 2388 2389 // Check if we can load the element with one instruction 2390 if (LdWidth <= NewVTWidth) { 2391 if (!NewVT.isVector()) { 2392 unsigned NumElts = WidenWidth / NewVTWidth; 2393 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2394 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2395 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp); 2396 } 2397 if (NewVT == WidenVT) 2398 return LdOp; 2399 2400 assert(WidenWidth % NewVTWidth == 0); 2401 unsigned NumConcat = WidenWidth / NewVTWidth; 2402 SmallVector<SDValue, 16> ConcatOps(NumConcat); 2403 SDValue UndefVal = DAG.getUNDEF(NewVT); 2404 ConcatOps[0] = LdOp; 2405 for (unsigned i = 1; i != NumConcat; ++i) 2406 ConcatOps[i] = UndefVal; 2407 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &ConcatOps[0], 2408 NumConcat); 2409 } 2410 2411 // Load vector by using multiple loads from largest vector to scalar 2412 SmallVector<SDValue, 16> LdOps; 2413 LdOps.push_back(LdOp); 2414 2415 LdWidth -= NewVTWidth; 2416 unsigned Offset = 0; 2417 2418 while (LdWidth > 0) { 2419 unsigned Increment = NewVTWidth / 8; 2420 Offset += Increment; 2421 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2422 DAG.getIntPtrConstant(Increment)); 2423 2424 SDValue L; 2425 if (LdWidth < NewVTWidth) { 2426 // Our current type we are using is too large, find a better size 2427 NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2428 NewVTWidth = NewVT.getSizeInBits(); 2429 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 2430 LD->getPointerInfo().getWithOffset(Offset), isVolatile, 2431 isNonTemporal, isInvariant, MinAlign(Align, Increment)); 2432 LdChain.push_back(L.getValue(1)); 2433 if (L->getValueType(0).isVector()) { 2434 SmallVector<SDValue, 16> Loads; 2435 Loads.push_back(L); 2436 unsigned size = L->getValueSizeInBits(0); 2437 while (size < LdOp->getValueSizeInBits(0)) { 2438 Loads.push_back(DAG.getUNDEF(L->getValueType(0))); 2439 size += L->getValueSizeInBits(0); 2440 } 2441 L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), 2442 &Loads[0], Loads.size()); 2443 } 2444 } else { 2445 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 2446 LD->getPointerInfo().getWithOffset(Offset), isVolatile, 2447 isNonTemporal, isInvariant, MinAlign(Align, Increment)); 2448 LdChain.push_back(L.getValue(1)); 2449 } 2450 2451 LdOps.push_back(L); 2452 2453 2454 LdWidth -= NewVTWidth; 2455 } 2456 2457 // Build the vector from the loads operations 2458 unsigned End = LdOps.size(); 2459 if (!LdOps[0].getValueType().isVector()) 2460 // All the loads are scalar loads. 2461 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End); 2462 2463 // If the load contains vectors, build the vector using concat vector. 2464 // All of the vectors used to loads are power of 2 and the scalars load 2465 // can be combined to make a power of 2 vector. 2466 SmallVector<SDValue, 16> ConcatOps(End); 2467 int i = End - 1; 2468 int Idx = End; 2469 EVT LdTy = LdOps[i].getValueType(); 2470 // First combine the scalar loads to a vector 2471 if (!LdTy.isVector()) { 2472 for (--i; i >= 0; --i) { 2473 LdTy = LdOps[i].getValueType(); 2474 if (LdTy.isVector()) 2475 break; 2476 } 2477 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i+1, End); 2478 } 2479 ConcatOps[--Idx] = LdOps[i]; 2480 for (--i; i >= 0; --i) { 2481 EVT NewLdTy = LdOps[i].getValueType(); 2482 if (NewLdTy != LdTy) { 2483 // Create a larger vector 2484 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy, 2485 &ConcatOps[Idx], End - Idx); 2486 Idx = End - 1; 2487 LdTy = NewLdTy; 2488 } 2489 ConcatOps[--Idx] = LdOps[i]; 2490 } 2491 2492 if (WidenWidth == LdTy.getSizeInBits()*(End - Idx)) 2493 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 2494 &ConcatOps[Idx], End - Idx); 2495 2496 // We need to fill the rest with undefs to build the vector 2497 unsigned NumOps = WidenWidth / LdTy.getSizeInBits(); 2498 SmallVector<SDValue, 16> WidenOps(NumOps); 2499 SDValue UndefVal = DAG.getUNDEF(LdTy); 2500 { 2501 unsigned i = 0; 2502 for (; i != End-Idx; ++i) 2503 WidenOps[i] = ConcatOps[Idx+i]; 2504 for (; i != NumOps; ++i) 2505 WidenOps[i] = UndefVal; 2506 } 2507 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &WidenOps[0],NumOps); 2508 } 2509 2510 SDValue 2511 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVector<SDValue, 16>& LdChain, 2512 LoadSDNode * LD, 2513 ISD::LoadExtType ExtType) { 2514 // For extension loads, it may not be more efficient to chop up the vector 2515 // and then extended it. Instead, we unroll the load and build a new vector. 2516 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2517 EVT LdVT = LD->getMemoryVT(); 2518 DebugLoc dl = LD->getDebugLoc(); 2519 assert(LdVT.isVector() && WidenVT.isVector()); 2520 2521 // Load information 2522 SDValue Chain = LD->getChain(); 2523 SDValue BasePtr = LD->getBasePtr(); 2524 unsigned Align = LD->getAlignment(); 2525 bool isVolatile = LD->isVolatile(); 2526 bool isNonTemporal = LD->isNonTemporal(); 2527 2528 EVT EltVT = WidenVT.getVectorElementType(); 2529 EVT LdEltVT = LdVT.getVectorElementType(); 2530 unsigned NumElts = LdVT.getVectorNumElements(); 2531 2532 // Load each element and widen 2533 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2534 SmallVector<SDValue, 16> Ops(WidenNumElts); 2535 unsigned Increment = LdEltVT.getSizeInBits() / 8; 2536 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, 2537 LD->getPointerInfo(), 2538 LdEltVT, isVolatile, isNonTemporal, Align); 2539 LdChain.push_back(Ops[0].getValue(1)); 2540 unsigned i = 0, Offset = Increment; 2541 for (i=1; i < NumElts; ++i, Offset += Increment) { 2542 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 2543 BasePtr, DAG.getIntPtrConstant(Offset)); 2544 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, 2545 LD->getPointerInfo().getWithOffset(Offset), LdEltVT, 2546 isVolatile, isNonTemporal, Align); 2547 LdChain.push_back(Ops[i].getValue(1)); 2548 } 2549 2550 // Fill the rest with undefs 2551 SDValue UndefVal = DAG.getUNDEF(EltVT); 2552 for (; i != WidenNumElts; ++i) 2553 Ops[i] = UndefVal; 2554 2555 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size()); 2556 } 2557 2558 2559 void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain, 2560 StoreSDNode *ST) { 2561 // The strategy assumes that we can efficiently store powers of two widths. 2562 // The routines chops the vector into the largest vector stores with the same 2563 // element type or scalar stores. 2564 SDValue Chain = ST->getChain(); 2565 SDValue BasePtr = ST->getBasePtr(); 2566 unsigned Align = ST->getAlignment(); 2567 bool isVolatile = ST->isVolatile(); 2568 bool isNonTemporal = ST->isNonTemporal(); 2569 SDValue ValOp = GetWidenedVector(ST->getValue()); 2570 DebugLoc dl = ST->getDebugLoc(); 2571 2572 EVT StVT = ST->getMemoryVT(); 2573 unsigned StWidth = StVT.getSizeInBits(); 2574 EVT ValVT = ValOp.getValueType(); 2575 unsigned ValWidth = ValVT.getSizeInBits(); 2576 EVT ValEltVT = ValVT.getVectorElementType(); 2577 unsigned ValEltWidth = ValEltVT.getSizeInBits(); 2578 assert(StVT.getVectorElementType() == ValEltVT); 2579 2580 int Idx = 0; // current index to store 2581 unsigned Offset = 0; // offset from base to store 2582 while (StWidth != 0) { 2583 // Find the largest vector type we can store with 2584 EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT); 2585 unsigned NewVTWidth = NewVT.getSizeInBits(); 2586 unsigned Increment = NewVTWidth / 8; 2587 if (NewVT.isVector()) { 2588 unsigned NumVTElts = NewVT.getVectorNumElements(); 2589 do { 2590 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp, 2591 DAG.getIntPtrConstant(Idx)); 2592 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 2593 ST->getPointerInfo().getWithOffset(Offset), 2594 isVolatile, isNonTemporal, 2595 MinAlign(Align, Offset))); 2596 StWidth -= NewVTWidth; 2597 Offset += Increment; 2598 Idx += NumVTElts; 2599 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2600 DAG.getIntPtrConstant(Increment)); 2601 } while (StWidth != 0 && StWidth >= NewVTWidth); 2602 } else { 2603 // Cast the vector to the scalar type we can store 2604 unsigned NumElts = ValWidth / NewVTWidth; 2605 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2606 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp); 2607 // Readjust index position based on new vector type 2608 Idx = Idx * ValEltWidth / NewVTWidth; 2609 do { 2610 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp, 2611 DAG.getIntPtrConstant(Idx++)); 2612 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 2613 ST->getPointerInfo().getWithOffset(Offset), 2614 isVolatile, isNonTemporal, 2615 MinAlign(Align, Offset))); 2616 StWidth -= NewVTWidth; 2617 Offset += Increment; 2618 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2619 DAG.getIntPtrConstant(Increment)); 2620 } while (StWidth != 0 && StWidth >= NewVTWidth); 2621 // Restore index back to be relative to the original widen element type 2622 Idx = Idx * NewVTWidth / ValEltWidth; 2623 } 2624 } 2625 } 2626 2627 void 2628 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVector<SDValue, 16>& StChain, 2629 StoreSDNode *ST) { 2630 // For extension loads, it may not be more efficient to truncate the vector 2631 // and then store it. Instead, we extract each element and then store it. 2632 SDValue Chain = ST->getChain(); 2633 SDValue BasePtr = ST->getBasePtr(); 2634 unsigned Align = ST->getAlignment(); 2635 bool isVolatile = ST->isVolatile(); 2636 bool isNonTemporal = ST->isNonTemporal(); 2637 SDValue ValOp = GetWidenedVector(ST->getValue()); 2638 DebugLoc dl = ST->getDebugLoc(); 2639 2640 EVT StVT = ST->getMemoryVT(); 2641 EVT ValVT = ValOp.getValueType(); 2642 2643 // It must be true that we the widen vector type is bigger than where 2644 // we need to store. 2645 assert(StVT.isVector() && ValOp.getValueType().isVector()); 2646 assert(StVT.bitsLT(ValOp.getValueType())); 2647 2648 // For truncating stores, we can not play the tricks of chopping legal 2649 // vector types and bit cast it to the right type. Instead, we unroll 2650 // the store. 2651 EVT StEltVT = StVT.getVectorElementType(); 2652 EVT ValEltVT = ValVT.getVectorElementType(); 2653 unsigned Increment = ValEltVT.getSizeInBits() / 8; 2654 unsigned NumElts = StVT.getVectorNumElements(); 2655 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 2656 DAG.getIntPtrConstant(0)); 2657 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, 2658 ST->getPointerInfo(), StEltVT, 2659 isVolatile, isNonTemporal, Align)); 2660 unsigned Offset = Increment; 2661 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 2662 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 2663 BasePtr, DAG.getIntPtrConstant(Offset)); 2664 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 2665 DAG.getIntPtrConstant(0)); 2666 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, 2667 ST->getPointerInfo().getWithOffset(Offset), 2668 StEltVT, isVolatile, isNonTemporal, 2669 MinAlign(Align, Offset))); 2670 } 2671 } 2672 2673 /// Modifies a vector input (widen or narrows) to a vector of NVT. The 2674 /// input vector must have the same element type as NVT. 2675 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) { 2676 // Note that InOp might have been widened so it might already have 2677 // the right width or it might need be narrowed. 2678 EVT InVT = InOp.getValueType(); 2679 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 2680 "input and widen element type must match"); 2681 DebugLoc dl = InOp.getDebugLoc(); 2682 2683 // Check if InOp already has the right width. 2684 if (InVT == NVT) 2685 return InOp; 2686 2687 unsigned InNumElts = InVT.getVectorNumElements(); 2688 unsigned WidenNumElts = NVT.getVectorNumElements(); 2689 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 2690 unsigned NumConcat = WidenNumElts / InNumElts; 2691 SmallVector<SDValue, 16> Ops(NumConcat); 2692 SDValue UndefVal = DAG.getUNDEF(InVT); 2693 Ops[0] = InOp; 2694 for (unsigned i = 1; i != NumConcat; ++i) 2695 Ops[i] = UndefVal; 2696 2697 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat); 2698 } 2699 2700 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 2701 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 2702 DAG.getIntPtrConstant(0)); 2703 2704 // Fall back to extract and build. 2705 SmallVector<SDValue, 16> Ops(WidenNumElts); 2706 EVT EltVT = NVT.getVectorElementType(); 2707 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 2708 unsigned Idx; 2709 for (Idx = 0; Idx < MinNumElts; ++Idx) 2710 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2711 DAG.getIntPtrConstant(Idx)); 2712 2713 SDValue UndefVal = DAG.getUNDEF(EltVT); 2714 for ( ; Idx < WidenNumElts; ++Idx) 2715 Ops[Idx] = UndefVal; 2716 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts); 2717 } 2718