1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file performs vector type splitting and scalarization for LegalizeTypes. 11 // Scalarization is the act of changing a computation in an illegal one-element 12 // vector type to be a computation in its scalar element type. For example, 13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which 15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16 // types. 17 // Splitting is the act of changing a computation in an invalid vector type to 18 // be a computation in two vectors of half the size. For example, implementing 19 // <128 x f32> operations in terms of two <64 x f32> operations. 20 // 21 //===----------------------------------------------------------------------===// 22 23 #include "LegalizeTypes.h" 24 #include "llvm/IR/DataLayout.h" 25 #include "llvm/Support/ErrorHandling.h" 26 #include "llvm/Support/raw_ostream.h" 27 using namespace llvm; 28 29 //===----------------------------------------------------------------------===// 30 // Result Vector Scalarization: <1 x ty> -> ty. 31 //===----------------------------------------------------------------------===// 32 33 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 34 DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; 35 N->dump(&DAG); 36 dbgs() << "\n"); 37 SDValue R = SDValue(); 38 39 switch (N->getOpcode()) { 40 default: 41 #ifndef NDEBUG 42 dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 43 N->dump(&DAG); 44 dbgs() << "\n"; 45 #endif 46 report_fatal_error("Do not know how to scalarize the result of this " 47 "operator!\n"); 48 49 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; 50 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; 51 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; 52 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 53 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 54 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; 55 case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break; 56 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 57 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 58 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 59 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 60 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 61 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break; 62 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 63 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 64 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 65 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 66 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 67 case ISD::ANY_EXTEND: 68 case ISD::CTLZ: 69 case ISD::CTPOP: 70 case ISD::CTTZ: 71 case ISD::FABS: 72 case ISD::FCEIL: 73 case ISD::FCOS: 74 case ISD::FEXP: 75 case ISD::FEXP2: 76 case ISD::FFLOOR: 77 case ISD::FLOG: 78 case ISD::FLOG10: 79 case ISD::FLOG2: 80 case ISD::FNEARBYINT: 81 case ISD::FNEG: 82 case ISD::FP_EXTEND: 83 case ISD::FP_TO_SINT: 84 case ISD::FP_TO_UINT: 85 case ISD::FRINT: 86 case ISD::FSIN: 87 case ISD::FSQRT: 88 case ISD::FTRUNC: 89 case ISD::SIGN_EXTEND: 90 case ISD::SINT_TO_FP: 91 case ISD::TRUNCATE: 92 case ISD::UINT_TO_FP: 93 case ISD::ZERO_EXTEND: 94 R = ScalarizeVecRes_UnaryOp(N); 95 break; 96 97 case ISD::ADD: 98 case ISD::AND: 99 case ISD::FADD: 100 case ISD::FDIV: 101 case ISD::FMUL: 102 case ISD::FPOW: 103 case ISD::FREM: 104 case ISD::FSUB: 105 case ISD::MUL: 106 case ISD::OR: 107 case ISD::SDIV: 108 case ISD::SREM: 109 case ISD::SUB: 110 case ISD::UDIV: 111 case ISD::UREM: 112 case ISD::XOR: 113 case ISD::SHL: 114 case ISD::SRA: 115 case ISD::SRL: 116 R = ScalarizeVecRes_BinOp(N); 117 break; 118 case ISD::FMA: 119 R = ScalarizeVecRes_TernaryOp(N); 120 break; 121 } 122 123 // If R is null, the sub-method took care of registering the result. 124 if (R.getNode()) 125 SetScalarizedVector(SDValue(N, ResNo), R); 126 } 127 128 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 129 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 130 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 131 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 132 LHS.getValueType(), LHS, RHS); 133 } 134 135 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { 136 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 137 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 138 SDValue Op2 = GetScalarizedVector(N->getOperand(2)); 139 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 140 Op0.getValueType(), Op0, Op1, Op2); 141 } 142 143 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, 144 unsigned ResNo) { 145 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 146 return GetScalarizedVector(Op); 147 } 148 149 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { 150 EVT NewVT = N->getValueType(0).getVectorElementType(); 151 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), 152 NewVT, N->getOperand(0)); 153 } 154 155 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { 156 EVT EltVT = N->getValueType(0).getVectorElementType(); 157 SDValue InOp = N->getOperand(0); 158 // The BUILD_VECTOR operands may be of wider element types and 159 // we may need to truncate them back to the requested return type. 160 if (EltVT.isInteger()) 161 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 162 return InOp; 163 } 164 165 SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 166 EVT NewVT = N->getValueType(0).getVectorElementType(); 167 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 168 return DAG.getConvertRndSat(NewVT, N->getDebugLoc(), 169 Op0, DAG.getValueType(NewVT), 170 DAG.getValueType(Op0.getValueType()), 171 N->getOperand(3), 172 N->getOperand(4), 173 cast<CvtRndSatSDNode>(N)->getCvtCode()); 174 } 175 176 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 177 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 178 N->getValueType(0).getVectorElementType(), 179 N->getOperand(0), N->getOperand(1)); 180 } 181 182 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { 183 EVT NewVT = N->getValueType(0).getVectorElementType(); 184 SDValue Op = GetScalarizedVector(N->getOperand(0)); 185 return DAG.getNode(ISD::FP_ROUND, N->getDebugLoc(), 186 NewVT, Op, N->getOperand(1)); 187 } 188 189 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 190 SDValue Op = GetScalarizedVector(N->getOperand(0)); 191 return DAG.getNode(ISD::FPOWI, N->getDebugLoc(), 192 Op.getValueType(), Op, N->getOperand(1)); 193 } 194 195 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 196 // The value to insert may have a wider type than the vector element type, 197 // so be sure to truncate it to the element type if necessary. 198 SDValue Op = N->getOperand(1); 199 EVT EltVT = N->getValueType(0).getVectorElementType(); 200 if (Op.getValueType() != EltVT) 201 // FIXME: Can this happen for floating point types? 202 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op); 203 return Op; 204 } 205 206 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 207 assert(N->isUnindexed() && "Indexed vector load?"); 208 209 SDValue Result = DAG.getLoad(ISD::UNINDEXED, 210 N->getExtensionType(), 211 N->getValueType(0).getVectorElementType(), 212 N->getDebugLoc(), 213 N->getChain(), N->getBasePtr(), 214 DAG.getUNDEF(N->getBasePtr().getValueType()), 215 N->getPointerInfo(), 216 N->getMemoryVT().getVectorElementType(), 217 N->isVolatile(), N->isNonTemporal(), 218 N->isInvariant(), N->getOriginalAlignment()); 219 220 // Legalized the chain result - switch anything that used the old chain to 221 // use the new one. 222 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 223 return Result; 224 } 225 226 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 227 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 228 EVT DestVT = N->getValueType(0).getVectorElementType(); 229 SDValue Op = GetScalarizedVector(N->getOperand(0)); 230 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op); 231 } 232 233 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 234 EVT EltVT = N->getValueType(0).getVectorElementType(); 235 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 236 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 237 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), EltVT, 238 LHS, DAG.getValueType(ExtVT)); 239 } 240 241 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 242 // If the operand is wider than the vector element type then it is implicitly 243 // truncated. Make that explicit here. 244 EVT EltVT = N->getValueType(0).getVectorElementType(); 245 SDValue InOp = N->getOperand(0); 246 if (InOp.getValueType() != EltVT) 247 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 248 return InOp; 249 } 250 251 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { 252 SDValue Cond = GetScalarizedVector(N->getOperand(0)); 253 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 254 TargetLowering::BooleanContent ScalarBool = TLI.getBooleanContents(false); 255 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true); 256 if (ScalarBool != VecBool) { 257 EVT CondVT = Cond.getValueType(); 258 switch (ScalarBool) { 259 case TargetLowering::UndefinedBooleanContent: 260 break; 261 case TargetLowering::ZeroOrOneBooleanContent: 262 assert(VecBool == TargetLowering::UndefinedBooleanContent || 263 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent); 264 // Vector read from all ones, scalar expects a single 1 so mask. 265 Cond = DAG.getNode(ISD::AND, N->getDebugLoc(), CondVT, 266 Cond, DAG.getConstant(1, CondVT)); 267 break; 268 case TargetLowering::ZeroOrNegativeOneBooleanContent: 269 assert(VecBool == TargetLowering::UndefinedBooleanContent || 270 VecBool == TargetLowering::ZeroOrOneBooleanContent); 271 // Vector reads from a one, scalar from all ones so sign extend. 272 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), CondVT, 273 Cond, DAG.getValueType(MVT::i1)); 274 break; 275 } 276 } 277 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 278 LHS.getValueType(), Cond, LHS, 279 GetScalarizedVector(N->getOperand(2))); 280 } 281 282 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 283 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 284 return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 285 LHS.getValueType(), N->getOperand(0), LHS, 286 GetScalarizedVector(N->getOperand(2))); 287 } 288 289 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 290 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 291 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(), 292 N->getOperand(0), N->getOperand(1), 293 LHS, GetScalarizedVector(N->getOperand(3)), 294 N->getOperand(4)); 295 } 296 297 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 298 assert(N->getValueType(0).isVector() == 299 N->getOperand(0).getValueType().isVector() && 300 "Scalar/Vector type mismatch"); 301 302 if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N); 303 304 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 305 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 306 DebugLoc DL = N->getDebugLoc(); 307 308 // Turn it into a scalar SETCC. 309 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 310 } 311 312 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 313 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 314 } 315 316 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 317 // Figure out if the scalar is the LHS or RHS and return it. 318 SDValue Arg = N->getOperand(2).getOperand(0); 319 if (Arg.getOpcode() == ISD::UNDEF) 320 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 321 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 322 return GetScalarizedVector(N->getOperand(Op)); 323 } 324 325 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 326 assert(N->getValueType(0).isVector() && 327 N->getOperand(0).getValueType().isVector() && 328 "Operand types must be vectors"); 329 330 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 331 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 332 EVT NVT = N->getValueType(0).getVectorElementType(); 333 DebugLoc DL = N->getDebugLoc(); 334 335 // Turn it into a scalar SETCC. 336 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 337 N->getOperand(2)); 338 // Vectors may have a different boolean contents to scalars. Promote the 339 // value appropriately. 340 ISD::NodeType ExtendCode = 341 TargetLowering::getExtendForContent(TLI.getBooleanContents(true)); 342 return DAG.getNode(ExtendCode, DL, NVT, Res); 343 } 344 345 346 //===----------------------------------------------------------------------===// 347 // Operand Vector Scalarization <1 x ty> -> ty. 348 //===----------------------------------------------------------------------===// 349 350 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 351 DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; 352 N->dump(&DAG); 353 dbgs() << "\n"); 354 SDValue Res = SDValue(); 355 356 if (Res.getNode() == 0) { 357 switch (N->getOpcode()) { 358 default: 359 #ifndef NDEBUG 360 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 361 N->dump(&DAG); 362 dbgs() << "\n"; 363 #endif 364 llvm_unreachable("Do not know how to scalarize this operator's operand!"); 365 case ISD::BITCAST: 366 Res = ScalarizeVecOp_BITCAST(N); 367 break; 368 case ISD::ANY_EXTEND: 369 case ISD::ZERO_EXTEND: 370 case ISD::SIGN_EXTEND: 371 Res = ScalarizeVecOp_EXTEND(N); 372 break; 373 case ISD::CONCAT_VECTORS: 374 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 375 break; 376 case ISD::EXTRACT_VECTOR_ELT: 377 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 378 break; 379 case ISD::STORE: 380 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 381 break; 382 } 383 } 384 385 // If the result is null, the sub-method took care of registering results etc. 386 if (!Res.getNode()) return false; 387 388 // If the result is N, the sub-method updated N in place. Tell the legalizer 389 // core about this. 390 if (Res.getNode() == N) 391 return true; 392 393 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 394 "Invalid operand expansion"); 395 396 ReplaceValueWith(SDValue(N, 0), Res); 397 return false; 398 } 399 400 /// ScalarizeVecOp_BITCAST - If the value to convert is a vector that needs 401 /// to be scalarized, it must be <1 x ty>. Convert the element instead. 402 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { 403 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 404 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), 405 N->getValueType(0), Elt); 406 } 407 408 /// ScalarizeVecOp_EXTEND - If the value to extend is a vector that needs 409 /// to be scalarized, it must be <1 x ty>. Extend the element instead. 410 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTEND(SDNode *N) { 411 assert(N->getValueType(0).getVectorNumElements() == 1 && 412 "Unexected vector type!"); 413 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 414 SmallVector<SDValue, 1> Ops(1); 415 Ops[0] = DAG.getNode(N->getOpcode(), N->getDebugLoc(), 416 N->getValueType(0).getScalarType(), Elt); 417 // Revectorize the result so the types line up with what the uses of this 418 // expression expect. 419 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 420 &Ops[0], 1); 421 } 422 423 /// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 424 /// use a BUILD_VECTOR instead. 425 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 426 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 427 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 428 Ops[i] = GetScalarizedVector(N->getOperand(i)); 429 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 430 &Ops[0], Ops.size()); 431 } 432 433 /// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 434 /// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 435 /// index. 436 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 437 SDValue Res = GetScalarizedVector(N->getOperand(0)); 438 if (Res.getValueType() != N->getValueType(0)) 439 Res = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), 440 Res); 441 return Res; 442 } 443 444 /// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 445 /// scalarized, it must be <1 x ty>. Just store the element. 446 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 447 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 448 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 449 DebugLoc dl = N->getDebugLoc(); 450 451 if (N->isTruncatingStore()) 452 return DAG.getTruncStore(N->getChain(), dl, 453 GetScalarizedVector(N->getOperand(1)), 454 N->getBasePtr(), N->getPointerInfo(), 455 N->getMemoryVT().getVectorElementType(), 456 N->isVolatile(), N->isNonTemporal(), 457 N->getAlignment()); 458 459 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 460 N->getBasePtr(), N->getPointerInfo(), 461 N->isVolatile(), N->isNonTemporal(), 462 N->getOriginalAlignment()); 463 } 464 465 466 //===----------------------------------------------------------------------===// 467 // Result Vector Splitting 468 //===----------------------------------------------------------------------===// 469 470 /// SplitVectorResult - This method is called when the specified result of the 471 /// specified node is found to need vector splitting. At this point, the node 472 /// may also have invalid operands or may have other results that need 473 /// legalization, we just know that (at least) one result needs vector 474 /// splitting. 475 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 476 DEBUG(dbgs() << "Split node result: "; 477 N->dump(&DAG); 478 dbgs() << "\n"); 479 SDValue Lo, Hi; 480 481 // See if the target wants to custom expand this node. 482 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 483 return; 484 485 switch (N->getOpcode()) { 486 default: 487 #ifndef NDEBUG 488 dbgs() << "SplitVectorResult #" << ResNo << ": "; 489 N->dump(&DAG); 490 dbgs() << "\n"; 491 #endif 492 report_fatal_error("Do not know how to split the result of this " 493 "operator!\n"); 494 495 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 496 case ISD::VSELECT: 497 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 498 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 499 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 500 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break; 501 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 502 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 503 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 504 case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 505 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 506 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 507 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 508 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 509 case ISD::LOAD: 510 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 511 break; 512 case ISD::SETCC: 513 SplitVecRes_SETCC(N, Lo, Hi); 514 break; 515 case ISD::VECTOR_SHUFFLE: 516 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 517 break; 518 519 case ISD::ANY_EXTEND: 520 case ISD::CONVERT_RNDSAT: 521 case ISD::CTLZ: 522 case ISD::CTTZ: 523 case ISD::CTLZ_ZERO_UNDEF: 524 case ISD::CTTZ_ZERO_UNDEF: 525 case ISD::CTPOP: 526 case ISD::FABS: 527 case ISD::FCEIL: 528 case ISD::FCOS: 529 case ISD::FEXP: 530 case ISD::FEXP2: 531 case ISD::FFLOOR: 532 case ISD::FLOG: 533 case ISD::FLOG10: 534 case ISD::FLOG2: 535 case ISD::FNEARBYINT: 536 case ISD::FNEG: 537 case ISD::FP_EXTEND: 538 case ISD::FP_ROUND: 539 case ISD::FP_TO_SINT: 540 case ISD::FP_TO_UINT: 541 case ISD::FRINT: 542 case ISD::FSIN: 543 case ISD::FSQRT: 544 case ISD::FTRUNC: 545 case ISD::SIGN_EXTEND: 546 case ISD::SINT_TO_FP: 547 case ISD::TRUNCATE: 548 case ISD::UINT_TO_FP: 549 case ISD::ZERO_EXTEND: 550 SplitVecRes_UnaryOp(N, Lo, Hi); 551 break; 552 553 case ISD::ADD: 554 case ISD::SUB: 555 case ISD::MUL: 556 case ISD::FADD: 557 case ISD::FSUB: 558 case ISD::FMUL: 559 case ISD::SDIV: 560 case ISD::UDIV: 561 case ISD::FDIV: 562 case ISD::FPOW: 563 case ISD::AND: 564 case ISD::OR: 565 case ISD::XOR: 566 case ISD::SHL: 567 case ISD::SRA: 568 case ISD::SRL: 569 case ISD::UREM: 570 case ISD::SREM: 571 case ISD::FREM: 572 SplitVecRes_BinOp(N, Lo, Hi); 573 break; 574 case ISD::FMA: 575 SplitVecRes_TernaryOp(N, Lo, Hi); 576 break; 577 } 578 579 // If Lo/Hi is null, the sub-method took care of registering results etc. 580 if (Lo.getNode()) 581 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 582 } 583 584 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 585 SDValue &Hi) { 586 SDValue LHSLo, LHSHi; 587 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 588 SDValue RHSLo, RHSHi; 589 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 590 DebugLoc dl = N->getDebugLoc(); 591 592 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 593 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 594 } 595 596 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, 597 SDValue &Hi) { 598 SDValue Op0Lo, Op0Hi; 599 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 600 SDValue Op1Lo, Op1Hi; 601 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 602 SDValue Op2Lo, Op2Hi; 603 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi); 604 DebugLoc dl = N->getDebugLoc(); 605 606 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), 607 Op0Lo, Op1Lo, Op2Lo); 608 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), 609 Op0Hi, Op1Hi, Op2Hi); 610 } 611 612 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, 613 SDValue &Hi) { 614 // We know the result is a vector. The input may be either a vector or a 615 // scalar value. 616 EVT LoVT, HiVT; 617 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 618 DebugLoc dl = N->getDebugLoc(); 619 620 SDValue InOp = N->getOperand(0); 621 EVT InVT = InOp.getValueType(); 622 623 // Handle some special cases efficiently. 624 switch (getTypeAction(InVT)) { 625 case TargetLowering::TypeLegal: 626 case TargetLowering::TypePromoteInteger: 627 case TargetLowering::TypeSoftenFloat: 628 case TargetLowering::TypeScalarizeVector: 629 case TargetLowering::TypeWidenVector: 630 break; 631 case TargetLowering::TypeExpandInteger: 632 case TargetLowering::TypeExpandFloat: 633 // A scalar to vector conversion, where the scalar needs expansion. 634 // If the vector is being split in two then we can just convert the 635 // expanded pieces. 636 if (LoVT == HiVT) { 637 GetExpandedOp(InOp, Lo, Hi); 638 if (TLI.isBigEndian()) 639 std::swap(Lo, Hi); 640 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 641 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 642 return; 643 } 644 break; 645 case TargetLowering::TypeSplitVector: 646 // If the input is a vector that needs to be split, convert each split 647 // piece of the input now. 648 GetSplitVector(InOp, Lo, Hi); 649 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 650 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 651 return; 652 } 653 654 // In the general case, convert the input to an integer and split it by hand. 655 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 656 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 657 if (TLI.isBigEndian()) 658 std::swap(LoIntVT, HiIntVT); 659 660 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 661 662 if (TLI.isBigEndian()) 663 std::swap(Lo, Hi); 664 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 665 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 666 } 667 668 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 669 SDValue &Hi) { 670 EVT LoVT, HiVT; 671 DebugLoc dl = N->getDebugLoc(); 672 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 673 unsigned LoNumElts = LoVT.getVectorNumElements(); 674 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 675 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size()); 676 677 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 678 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size()); 679 } 680 681 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 682 SDValue &Hi) { 683 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 684 DebugLoc dl = N->getDebugLoc(); 685 unsigned NumSubvectors = N->getNumOperands() / 2; 686 if (NumSubvectors == 1) { 687 Lo = N->getOperand(0); 688 Hi = N->getOperand(1); 689 return; 690 } 691 692 EVT LoVT, HiVT; 693 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 694 695 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 696 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size()); 697 698 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 699 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size()); 700 } 701 702 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 703 SDValue &Hi) { 704 SDValue Vec = N->getOperand(0); 705 SDValue Idx = N->getOperand(1); 706 DebugLoc dl = N->getDebugLoc(); 707 708 EVT LoVT, HiVT; 709 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 710 711 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 712 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 713 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, 714 DAG.getIntPtrConstant(IdxVal + LoVT.getVectorNumElements())); 715 } 716 717 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 718 SDValue &Hi) { 719 DebugLoc dl = N->getDebugLoc(); 720 GetSplitVector(N->getOperand(0), Lo, Hi); 721 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 722 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 723 } 724 725 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 726 SDValue &Hi) { 727 SDValue LHSLo, LHSHi; 728 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 729 DebugLoc dl = N->getDebugLoc(); 730 731 EVT LoVT, HiVT; 732 GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT(), LoVT, HiVT); 733 734 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 735 DAG.getValueType(LoVT)); 736 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 737 DAG.getValueType(HiVT)); 738 } 739 740 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 741 SDValue &Hi) { 742 SDValue Vec = N->getOperand(0); 743 SDValue Elt = N->getOperand(1); 744 SDValue Idx = N->getOperand(2); 745 DebugLoc dl = N->getDebugLoc(); 746 GetSplitVector(Vec, Lo, Hi); 747 748 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 749 unsigned IdxVal = CIdx->getZExtValue(); 750 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 751 if (IdxVal < LoNumElts) 752 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 753 Lo.getValueType(), Lo, Elt, Idx); 754 else 755 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 756 DAG.getIntPtrConstant(IdxVal - LoNumElts)); 757 return; 758 } 759 760 // Spill the vector to the stack. 761 EVT VecVT = Vec.getValueType(); 762 EVT EltVT = VecVT.getVectorElementType(); 763 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 764 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 765 MachinePointerInfo(), false, false, 0); 766 767 // Store the new element. This may be larger than the vector element type, 768 // so use a truncating store. 769 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 770 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 771 unsigned Alignment = 772 TLI.getDataLayout()->getPrefTypeAlignment(VecType); 773 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT, 774 false, false, 0); 775 776 // Load the Lo part from the stack slot. 777 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 778 false, false, false, 0); 779 780 // Increment the pointer to the other part. 781 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 782 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 783 DAG.getIntPtrConstant(IncrementSize)); 784 785 // Load the Hi part from the stack slot. 786 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 787 false, false, false, MinAlign(Alignment, IncrementSize)); 788 } 789 790 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 791 SDValue &Hi) { 792 EVT LoVT, HiVT; 793 DebugLoc dl = N->getDebugLoc(); 794 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 795 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 796 Hi = DAG.getUNDEF(HiVT); 797 } 798 799 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 800 SDValue &Hi) { 801 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 802 EVT LoVT, HiVT; 803 DebugLoc dl = LD->getDebugLoc(); 804 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 805 806 ISD::LoadExtType ExtType = LD->getExtensionType(); 807 SDValue Ch = LD->getChain(); 808 SDValue Ptr = LD->getBasePtr(); 809 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 810 EVT MemoryVT = LD->getMemoryVT(); 811 unsigned Alignment = LD->getOriginalAlignment(); 812 bool isVolatile = LD->isVolatile(); 813 bool isNonTemporal = LD->isNonTemporal(); 814 bool isInvariant = LD->isInvariant(); 815 816 EVT LoMemVT, HiMemVT; 817 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 818 819 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset, 820 LD->getPointerInfo(), LoMemVT, isVolatile, isNonTemporal, 821 isInvariant, Alignment); 822 823 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 824 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 825 DAG.getIntPtrConstant(IncrementSize)); 826 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, 827 LD->getPointerInfo().getWithOffset(IncrementSize), 828 HiMemVT, isVolatile, isNonTemporal, isInvariant, Alignment); 829 830 // Build a factor node to remember that this load is independent of the 831 // other one. 832 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 833 Hi.getValue(1)); 834 835 // Legalized the chain result - switch anything that used the old chain to 836 // use the new one. 837 ReplaceValueWith(SDValue(LD, 1), Ch); 838 } 839 840 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 841 assert(N->getValueType(0).isVector() && 842 N->getOperand(0).getValueType().isVector() && 843 "Operand types must be vectors"); 844 845 EVT LoVT, HiVT; 846 DebugLoc DL = N->getDebugLoc(); 847 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 848 849 // Split the input. 850 EVT InVT = N->getOperand(0).getValueType(); 851 SDValue LL, LH, RL, RH; 852 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 853 LoVT.getVectorNumElements()); 854 LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 855 DAG.getIntPtrConstant(0)); 856 LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 857 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 858 859 RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 860 DAG.getIntPtrConstant(0)); 861 RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 862 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 863 864 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 865 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 866 } 867 868 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 869 SDValue &Hi) { 870 // Get the dest types - they may not match the input types, e.g. int_to_fp. 871 EVT LoVT, HiVT; 872 DebugLoc dl = N->getDebugLoc(); 873 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 874 875 // If the input also splits, handle it directly for a compile time speedup. 876 // Otherwise split it by hand. 877 EVT InVT = N->getOperand(0).getValueType(); 878 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) { 879 GetSplitVector(N->getOperand(0), Lo, Hi); 880 } else { 881 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 882 LoVT.getVectorNumElements()); 883 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 884 DAG.getIntPtrConstant(0)); 885 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 886 DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 887 } 888 889 if (N->getOpcode() == ISD::FP_ROUND) { 890 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1)); 891 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1)); 892 } else if (N->getOpcode() == ISD::CONVERT_RNDSAT) { 893 SDValue DTyOpLo = DAG.getValueType(LoVT); 894 SDValue DTyOpHi = DAG.getValueType(HiVT); 895 SDValue STyOpLo = DAG.getValueType(Lo.getValueType()); 896 SDValue STyOpHi = DAG.getValueType(Hi.getValueType()); 897 SDValue RndOp = N->getOperand(3); 898 SDValue SatOp = N->getOperand(4); 899 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 900 Lo = DAG.getConvertRndSat(LoVT, dl, Lo, DTyOpLo, STyOpLo, RndOp, SatOp, 901 CvtCode); 902 Hi = DAG.getConvertRndSat(HiVT, dl, Hi, DTyOpHi, STyOpHi, RndOp, SatOp, 903 CvtCode); 904 } else { 905 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 906 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 907 } 908 } 909 910 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 911 SDValue &Lo, SDValue &Hi) { 912 // The low and high parts of the original input give four input vectors. 913 SDValue Inputs[4]; 914 DebugLoc dl = N->getDebugLoc(); 915 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 916 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 917 EVT NewVT = Inputs[0].getValueType(); 918 unsigned NewElts = NewVT.getVectorNumElements(); 919 920 // If Lo or Hi uses elements from at most two of the four input vectors, then 921 // express it as a vector shuffle of those two inputs. Otherwise extract the 922 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 923 SmallVector<int, 16> Ops; 924 for (unsigned High = 0; High < 2; ++High) { 925 SDValue &Output = High ? Hi : Lo; 926 927 // Build a shuffle mask for the output, discovering on the fly which 928 // input vectors to use as shuffle operands (recorded in InputUsed). 929 // If building a suitable shuffle vector proves too hard, then bail 930 // out with useBuildVector set. 931 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 932 unsigned FirstMaskIdx = High * NewElts; 933 bool useBuildVector = false; 934 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 935 // The mask element. This indexes into the input. 936 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 937 938 // The input vector this mask element indexes into. 939 unsigned Input = (unsigned)Idx / NewElts; 940 941 if (Input >= array_lengthof(Inputs)) { 942 // The mask element does not index into any input vector. 943 Ops.push_back(-1); 944 continue; 945 } 946 947 // Turn the index into an offset from the start of the input vector. 948 Idx -= Input * NewElts; 949 950 // Find or create a shuffle vector operand to hold this input. 951 unsigned OpNo; 952 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 953 if (InputUsed[OpNo] == Input) { 954 // This input vector is already an operand. 955 break; 956 } else if (InputUsed[OpNo] == -1U) { 957 // Create a new operand for this input vector. 958 InputUsed[OpNo] = Input; 959 break; 960 } 961 } 962 963 if (OpNo >= array_lengthof(InputUsed)) { 964 // More than two input vectors used! Give up on trying to create a 965 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 966 useBuildVector = true; 967 break; 968 } 969 970 // Add the mask index for the new shuffle vector. 971 Ops.push_back(Idx + OpNo * NewElts); 972 } 973 974 if (useBuildVector) { 975 EVT EltVT = NewVT.getVectorElementType(); 976 SmallVector<SDValue, 16> SVOps; 977 978 // Extract the input elements by hand. 979 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 980 // The mask element. This indexes into the input. 981 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 982 983 // The input vector this mask element indexes into. 984 unsigned Input = (unsigned)Idx / NewElts; 985 986 if (Input >= array_lengthof(Inputs)) { 987 // The mask element is "undef" or indexes off the end of the input. 988 SVOps.push_back(DAG.getUNDEF(EltVT)); 989 continue; 990 } 991 992 // Turn the index into an offset from the start of the input vector. 993 Idx -= Input * NewElts; 994 995 // Extract the vector element by hand. 996 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 997 Inputs[Input], DAG.getIntPtrConstant(Idx))); 998 } 999 1000 // Construct the Lo/Hi output using a BUILD_VECTOR. 1001 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size()); 1002 } else if (InputUsed[0] == -1U) { 1003 // No input vectors were used! The result is undefined. 1004 Output = DAG.getUNDEF(NewVT); 1005 } else { 1006 SDValue Op0 = Inputs[InputUsed[0]]; 1007 // If only one input was used, use an undefined vector for the other. 1008 SDValue Op1 = InputUsed[1] == -1U ? 1009 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 1010 // At least one input vector was used. Create a new shuffle vector. 1011 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 1012 } 1013 1014 Ops.clear(); 1015 } 1016 } 1017 1018 1019 //===----------------------------------------------------------------------===// 1020 // Operand Vector Splitting 1021 //===----------------------------------------------------------------------===// 1022 1023 /// SplitVectorOperand - This method is called when the specified operand of the 1024 /// specified node is found to need vector splitting. At this point, all of the 1025 /// result types of the node are known to be legal, but other operands of the 1026 /// node may need legalization as well as the specified one. 1027 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 1028 DEBUG(dbgs() << "Split node operand: "; 1029 N->dump(&DAG); 1030 dbgs() << "\n"); 1031 SDValue Res = SDValue(); 1032 1033 if (Res.getNode() == 0) { 1034 switch (N->getOpcode()) { 1035 default: 1036 #ifndef NDEBUG 1037 dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 1038 N->dump(&DAG); 1039 dbgs() << "\n"; 1040 #endif 1041 report_fatal_error("Do not know how to split this operator's " 1042 "operand!\n"); 1043 1044 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; 1045 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; 1046 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 1047 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 1048 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; 1049 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; 1050 case ISD::STORE: 1051 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 1052 break; 1053 case ISD::VSELECT: 1054 Res = SplitVecOp_VSELECT(N, OpNo); 1055 break; 1056 case ISD::CTTZ: 1057 case ISD::CTLZ: 1058 case ISD::CTPOP: 1059 case ISD::FP_EXTEND: 1060 case ISD::FP_TO_SINT: 1061 case ISD::FP_TO_UINT: 1062 case ISD::SINT_TO_FP: 1063 case ISD::UINT_TO_FP: 1064 case ISD::FTRUNC: 1065 case ISD::TRUNCATE: 1066 case ISD::SIGN_EXTEND: 1067 case ISD::ZERO_EXTEND: 1068 case ISD::ANY_EXTEND: 1069 Res = SplitVecOp_UnaryOp(N); 1070 break; 1071 } 1072 } 1073 1074 // If the result is null, the sub-method took care of registering results etc. 1075 if (!Res.getNode()) return false; 1076 1077 // If the result is N, the sub-method updated N in place. Tell the legalizer 1078 // core about this. 1079 if (Res.getNode() == N) 1080 return true; 1081 1082 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1083 "Invalid operand expansion"); 1084 1085 ReplaceValueWith(SDValue(N, 0), Res); 1086 return false; 1087 } 1088 1089 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { 1090 // The only possibility for an illegal operand is the mask, since result type 1091 // legalization would have handled this node already otherwise. 1092 assert(OpNo == 0 && "Illegal operand must be mask"); 1093 1094 SDValue Mask = N->getOperand(0); 1095 SDValue Src0 = N->getOperand(1); 1096 SDValue Src1 = N->getOperand(2); 1097 DebugLoc DL = N->getDebugLoc(); 1098 EVT MaskVT = Mask.getValueType(); 1099 assert(MaskVT.isVector() && "VSELECT without a vector mask?"); 1100 1101 SDValue Lo, Hi; 1102 GetSplitVector(N->getOperand(0), Lo, Hi); 1103 assert(Lo.getValueType() == Hi.getValueType() && 1104 "Lo and Hi have differing types");; 1105 1106 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 1107 unsigned HiNumElts = Hi.getValueType().getVectorNumElements(); 1108 assert(LoNumElts == HiNumElts && "Asymmetric vector split?"); 1109 1110 LLVMContext &Ctx = *DAG.getContext(); 1111 SDValue Zero = DAG.getIntPtrConstant(0); 1112 SDValue LoElts = DAG.getIntPtrConstant(LoNumElts); 1113 EVT Src0VT = Src0.getValueType(); 1114 EVT Src0EltTy = Src0VT.getVectorElementType(); 1115 EVT MaskEltTy = MaskVT.getVectorElementType(); 1116 1117 EVT LoOpVT = EVT::getVectorVT(Ctx, Src0EltTy, LoNumElts); 1118 EVT LoMaskVT = EVT::getVectorVT(Ctx, MaskEltTy, LoNumElts); 1119 EVT HiOpVT = EVT::getVectorVT(Ctx, Src0EltTy, HiNumElts); 1120 EVT HiMaskVT = EVT::getVectorVT(Ctx, MaskEltTy, HiNumElts); 1121 1122 SDValue LoOp0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoOpVT, Src0, Zero); 1123 SDValue LoOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoOpVT, Src1, Zero); 1124 1125 SDValue HiOp0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiOpVT, Src0, LoElts); 1126 SDValue HiOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiOpVT, Src1, LoElts); 1127 1128 SDValue LoMask = 1129 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoMaskVT, Mask, Zero); 1130 SDValue HiMask = 1131 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiMaskVT, Mask, LoElts); 1132 1133 SDValue LoSelect = 1134 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1); 1135 SDValue HiSelect = 1136 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1); 1137 1138 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect); 1139 } 1140 1141 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 1142 // The result has a legal vector type, but the input needs splitting. 1143 EVT ResVT = N->getValueType(0); 1144 SDValue Lo, Hi; 1145 DebugLoc dl = N->getDebugLoc(); 1146 GetSplitVector(N->getOperand(0), Lo, Hi); 1147 EVT InVT = Lo.getValueType(); 1148 1149 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1150 InVT.getVectorNumElements()); 1151 1152 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 1153 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 1154 1155 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 1156 } 1157 1158 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { 1159 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will 1160 // end up being split all the way down to individual components. Convert the 1161 // split pieces into integers and reassemble. 1162 SDValue Lo, Hi; 1163 GetSplitVector(N->getOperand(0), Lo, Hi); 1164 Lo = BitConvertToInteger(Lo); 1165 Hi = BitConvertToInteger(Hi); 1166 1167 if (TLI.isBigEndian()) 1168 std::swap(Lo, Hi); 1169 1170 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), N->getValueType(0), 1171 JoinIntegers(Lo, Hi)); 1172 } 1173 1174 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1175 // We know that the extracted result type is legal. 1176 EVT SubVT = N->getValueType(0); 1177 SDValue Idx = N->getOperand(1); 1178 DebugLoc dl = N->getDebugLoc(); 1179 SDValue Lo, Hi; 1180 GetSplitVector(N->getOperand(0), Lo, Hi); 1181 1182 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1183 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1184 1185 if (IdxVal < LoElts) { 1186 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1187 "Extracted subvector crosses vector split!"); 1188 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1189 } else { 1190 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1191 DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1192 } 1193 } 1194 1195 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1196 SDValue Vec = N->getOperand(0); 1197 SDValue Idx = N->getOperand(1); 1198 EVT VecVT = Vec.getValueType(); 1199 1200 if (isa<ConstantSDNode>(Idx)) { 1201 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1202 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1203 1204 SDValue Lo, Hi; 1205 GetSplitVector(Vec, Lo, Hi); 1206 1207 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1208 1209 if (IdxVal < LoElts) 1210 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0); 1211 return SDValue(DAG.UpdateNodeOperands(N, Hi, 1212 DAG.getConstant(IdxVal - LoElts, 1213 Idx.getValueType())), 0); 1214 } 1215 1216 // Store the vector to the stack. 1217 EVT EltVT = VecVT.getVectorElementType(); 1218 DebugLoc dl = N->getDebugLoc(); 1219 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1220 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1221 MachinePointerInfo(), false, false, 0); 1222 1223 // Load back the required element. 1224 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1225 return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 1226 MachinePointerInfo(), EltVT, false, false, 0); 1227 } 1228 1229 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1230 assert(N->isUnindexed() && "Indexed store of vector?"); 1231 assert(OpNo == 1 && "Can only split the stored value"); 1232 DebugLoc DL = N->getDebugLoc(); 1233 1234 bool isTruncating = N->isTruncatingStore(); 1235 SDValue Ch = N->getChain(); 1236 SDValue Ptr = N->getBasePtr(); 1237 EVT MemoryVT = N->getMemoryVT(); 1238 unsigned Alignment = N->getOriginalAlignment(); 1239 bool isVol = N->isVolatile(); 1240 bool isNT = N->isNonTemporal(); 1241 SDValue Lo, Hi; 1242 GetSplitVector(N->getOperand(1), Lo, Hi); 1243 1244 EVT LoMemVT, HiMemVT; 1245 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 1246 1247 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1248 1249 if (isTruncating) 1250 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1251 LoMemVT, isVol, isNT, Alignment); 1252 else 1253 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1254 isVol, isNT, Alignment); 1255 1256 // Increment the pointer to the other half. 1257 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 1258 DAG.getIntPtrConstant(IncrementSize)); 1259 1260 if (isTruncating) 1261 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, 1262 N->getPointerInfo().getWithOffset(IncrementSize), 1263 HiMemVT, isVol, isNT, Alignment); 1264 else 1265 Hi = DAG.getStore(Ch, DL, Hi, Ptr, 1266 N->getPointerInfo().getWithOffset(IncrementSize), 1267 isVol, isNT, Alignment); 1268 1269 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 1270 } 1271 1272 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { 1273 DebugLoc DL = N->getDebugLoc(); 1274 1275 // The input operands all must have the same type, and we know the result the 1276 // result type is valid. Convert this to a buildvector which extracts all the 1277 // input elements. 1278 // TODO: If the input elements are power-two vectors, we could convert this to 1279 // a new CONCAT_VECTORS node with elements that are half-wide. 1280 SmallVector<SDValue, 32> Elts; 1281 EVT EltVT = N->getValueType(0).getVectorElementType(); 1282 for (unsigned op = 0, e = N->getNumOperands(); op != e; ++op) { 1283 SDValue Op = N->getOperand(op); 1284 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements(); 1285 i != e; ++i) { 1286 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, 1287 Op, DAG.getIntPtrConstant(i))); 1288 1289 } 1290 } 1291 1292 return DAG.getNode(ISD::BUILD_VECTOR, DL, N->getValueType(0), 1293 &Elts[0], Elts.size()); 1294 } 1295 1296 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { 1297 assert(N->getValueType(0).isVector() && 1298 N->getOperand(0).getValueType().isVector() && 1299 "Operand types must be vectors"); 1300 // The result has a legal vector type, but the input needs splitting. 1301 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; 1302 DebugLoc DL = N->getDebugLoc(); 1303 GetSplitVector(N->getOperand(0), Lo0, Hi0); 1304 GetSplitVector(N->getOperand(1), Lo1, Hi1); 1305 unsigned PartElements = Lo0.getValueType().getVectorNumElements(); 1306 EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements); 1307 EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements); 1308 1309 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2)); 1310 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2)); 1311 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes); 1312 return PromoteTargetBoolean(Con, N->getValueType(0)); 1313 } 1314 1315 1316 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { 1317 // The result has a legal vector type, but the input needs splitting. 1318 EVT ResVT = N->getValueType(0); 1319 SDValue Lo, Hi; 1320 DebugLoc DL = N->getDebugLoc(); 1321 GetSplitVector(N->getOperand(0), Lo, Hi); 1322 EVT InVT = Lo.getValueType(); 1323 1324 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1325 InVT.getVectorNumElements()); 1326 1327 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); 1328 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); 1329 1330 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); 1331 } 1332 1333 1334 1335 //===----------------------------------------------------------------------===// 1336 // Result Vector Widening 1337 //===----------------------------------------------------------------------===// 1338 1339 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1340 DEBUG(dbgs() << "Widen node result " << ResNo << ": "; 1341 N->dump(&DAG); 1342 dbgs() << "\n"); 1343 1344 // See if the target wants to custom widen this node. 1345 if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 1346 return; 1347 1348 SDValue Res = SDValue(); 1349 switch (N->getOpcode()) { 1350 default: 1351 #ifndef NDEBUG 1352 dbgs() << "WidenVectorResult #" << ResNo << ": "; 1353 N->dump(&DAG); 1354 dbgs() << "\n"; 1355 #endif 1356 llvm_unreachable("Do not know how to widen the result of this operator!"); 1357 1358 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; 1359 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; 1360 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1361 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1362 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1363 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1364 case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break; 1365 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1366 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1367 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1368 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 1369 case ISD::VSELECT: 1370 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1371 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1372 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break; 1373 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1374 case ISD::VECTOR_SHUFFLE: 1375 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 1376 break; 1377 case ISD::ADD: 1378 case ISD::AND: 1379 case ISD::BSWAP: 1380 case ISD::FADD: 1381 case ISD::FCOPYSIGN: 1382 case ISD::FDIV: 1383 case ISD::FMUL: 1384 case ISD::FPOW: 1385 case ISD::FREM: 1386 case ISD::FSUB: 1387 case ISD::MUL: 1388 case ISD::MULHS: 1389 case ISD::MULHU: 1390 case ISD::OR: 1391 case ISD::SDIV: 1392 case ISD::SREM: 1393 case ISD::UDIV: 1394 case ISD::UREM: 1395 case ISD::SUB: 1396 case ISD::XOR: 1397 Res = WidenVecRes_Binary(N); 1398 break; 1399 1400 case ISD::FPOWI: 1401 Res = WidenVecRes_POWI(N); 1402 break; 1403 1404 case ISD::SHL: 1405 case ISD::SRA: 1406 case ISD::SRL: 1407 Res = WidenVecRes_Shift(N); 1408 break; 1409 1410 case ISD::ANY_EXTEND: 1411 case ISD::FP_EXTEND: 1412 case ISD::FP_ROUND: 1413 case ISD::FP_TO_SINT: 1414 case ISD::FP_TO_UINT: 1415 case ISD::SIGN_EXTEND: 1416 case ISD::SINT_TO_FP: 1417 case ISD::TRUNCATE: 1418 case ISD::UINT_TO_FP: 1419 case ISD::ZERO_EXTEND: 1420 Res = WidenVecRes_Convert(N); 1421 break; 1422 1423 case ISD::CTLZ: 1424 case ISD::CTPOP: 1425 case ISD::CTTZ: 1426 case ISD::FABS: 1427 case ISD::FCEIL: 1428 case ISD::FCOS: 1429 case ISD::FEXP: 1430 case ISD::FEXP2: 1431 case ISD::FFLOOR: 1432 case ISD::FLOG: 1433 case ISD::FLOG10: 1434 case ISD::FLOG2: 1435 case ISD::FNEARBYINT: 1436 case ISD::FNEG: 1437 case ISD::FRINT: 1438 case ISD::FSIN: 1439 case ISD::FSQRT: 1440 case ISD::FTRUNC: 1441 Res = WidenVecRes_Unary(N); 1442 break; 1443 case ISD::FMA: 1444 Res = WidenVecRes_Ternary(N); 1445 break; 1446 } 1447 1448 // If Res is null, the sub-method took care of registering the result. 1449 if (Res.getNode()) 1450 SetWidenedVector(SDValue(N, ResNo), Res); 1451 } 1452 1453 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { 1454 // Ternary op widening. 1455 DebugLoc dl = N->getDebugLoc(); 1456 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1457 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1458 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1459 SDValue InOp3 = GetWidenedVector(N->getOperand(2)); 1460 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3); 1461 } 1462 1463 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1464 // Binary op widening. 1465 unsigned Opcode = N->getOpcode(); 1466 DebugLoc dl = N->getDebugLoc(); 1467 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1468 EVT WidenEltVT = WidenVT.getVectorElementType(); 1469 EVT VT = WidenVT; 1470 unsigned NumElts = VT.getVectorNumElements(); 1471 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 1472 NumElts = NumElts / 2; 1473 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1474 } 1475 1476 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) { 1477 // Operation doesn't trap so just widen as normal. 1478 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1479 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1480 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2); 1481 } 1482 1483 // No legal vector version so unroll the vector operation and then widen. 1484 if (NumElts == 1) 1485 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 1486 1487 // Since the operation can trap, apply operation on the original vector. 1488 EVT MaxVT = VT; 1489 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1490 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1491 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 1492 1493 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 1494 unsigned ConcatEnd = 0; // Current ConcatOps index. 1495 int Idx = 0; // Current Idx into input vectors. 1496 1497 // NumElts := greatest legal vector size (at most WidenVT) 1498 // while (orig. vector has unhandled elements) { 1499 // take munches of size NumElts from the beginning and add to ConcatOps 1500 // NumElts := next smaller supported vector size or 1 1501 // } 1502 while (CurNumElts != 0) { 1503 while (CurNumElts >= NumElts) { 1504 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1, 1505 DAG.getIntPtrConstant(Idx)); 1506 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2, 1507 DAG.getIntPtrConstant(Idx)); 1508 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2); 1509 Idx += NumElts; 1510 CurNumElts -= NumElts; 1511 } 1512 do { 1513 NumElts = NumElts / 2; 1514 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1515 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 1516 1517 if (NumElts == 1) { 1518 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 1519 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1520 InOp1, DAG.getIntPtrConstant(Idx)); 1521 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1522 InOp2, DAG.getIntPtrConstant(Idx)); 1523 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT, 1524 EOp1, EOp2); 1525 } 1526 CurNumElts = 0; 1527 } 1528 } 1529 1530 // Check to see if we have a single operation with the widen type. 1531 if (ConcatEnd == 1) { 1532 VT = ConcatOps[0].getValueType(); 1533 if (VT == WidenVT) 1534 return ConcatOps[0]; 1535 } 1536 1537 // while (Some element of ConcatOps is not of type MaxVT) { 1538 // From the end of ConcatOps, collect elements of the same type and put 1539 // them into an op of the next larger supported type 1540 // } 1541 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { 1542 Idx = ConcatEnd - 1; 1543 VT = ConcatOps[Idx--].getValueType(); 1544 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) 1545 Idx--; 1546 1547 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1; 1548 EVT NextVT; 1549 do { 1550 NextSize *= 2; 1551 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize); 1552 } while (!TLI.isTypeLegal(NextVT)); 1553 1554 if (!VT.isVector()) { 1555 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT 1556 SDValue VecOp = DAG.getUNDEF(NextVT); 1557 unsigned NumToInsert = ConcatEnd - Idx - 1; 1558 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) { 1559 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, 1560 ConcatOps[OpIdx], DAG.getIntPtrConstant(i)); 1561 } 1562 ConcatOps[Idx+1] = VecOp; 1563 ConcatEnd = Idx + 2; 1564 } else { 1565 // Vector type, create a CONCAT_VECTORS of type NextVT 1566 SDValue undefVec = DAG.getUNDEF(VT); 1567 unsigned OpsToConcat = NextSize/VT.getVectorNumElements(); 1568 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat); 1569 unsigned RealVals = ConcatEnd - Idx - 1; 1570 unsigned SubConcatEnd = 0; 1571 unsigned SubConcatIdx = Idx + 1; 1572 while (SubConcatEnd < RealVals) 1573 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx]; 1574 while (SubConcatEnd < OpsToConcat) 1575 SubConcatOps[SubConcatEnd++] = undefVec; 1576 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1577 NextVT, &SubConcatOps[0], 1578 OpsToConcat); 1579 ConcatEnd = SubConcatIdx + 1; 1580 } 1581 } 1582 1583 // Check to see if we have a single operation with the widen type. 1584 if (ConcatEnd == 1) { 1585 VT = ConcatOps[0].getValueType(); 1586 if (VT == WidenVT) 1587 return ConcatOps[0]; 1588 } 1589 1590 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT 1591 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements(); 1592 if (NumOps != ConcatEnd ) { 1593 SDValue UndefVal = DAG.getUNDEF(MaxVT); 1594 for (unsigned j = ConcatEnd; j < NumOps; ++j) 1595 ConcatOps[j] = UndefVal; 1596 } 1597 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &ConcatOps[0], NumOps); 1598 } 1599 1600 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1601 SDValue InOp = N->getOperand(0); 1602 DebugLoc DL = N->getDebugLoc(); 1603 1604 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1605 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1606 1607 EVT InVT = InOp.getValueType(); 1608 EVT InEltVT = InVT.getVectorElementType(); 1609 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1610 1611 unsigned Opcode = N->getOpcode(); 1612 unsigned InVTNumElts = InVT.getVectorNumElements(); 1613 1614 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 1615 InOp = GetWidenedVector(N->getOperand(0)); 1616 InVT = InOp.getValueType(); 1617 InVTNumElts = InVT.getVectorNumElements(); 1618 if (InVTNumElts == WidenNumElts) { 1619 if (N->getNumOperands() == 1) 1620 return DAG.getNode(Opcode, DL, WidenVT, InOp); 1621 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1)); 1622 } 1623 } 1624 1625 if (TLI.isTypeLegal(InWidenVT)) { 1626 // Because the result and the input are different vector types, widening 1627 // the result could create a legal type but widening the input might make 1628 // it an illegal type that might lead to repeatedly splitting the input 1629 // and then widening it. To avoid this, we widen the input only if 1630 // it results in a legal type. 1631 if (WidenNumElts % InVTNumElts == 0) { 1632 // Widen the input and call convert on the widened input vector. 1633 unsigned NumConcat = WidenNumElts/InVTNumElts; 1634 SmallVector<SDValue, 16> Ops(NumConcat); 1635 Ops[0] = InOp; 1636 SDValue UndefVal = DAG.getUNDEF(InVT); 1637 for (unsigned i = 1; i != NumConcat; ++i) 1638 Ops[i] = UndefVal; 1639 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, 1640 &Ops[0], NumConcat); 1641 if (N->getNumOperands() == 1) 1642 return DAG.getNode(Opcode, DL, WidenVT, InVec); 1643 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1)); 1644 } 1645 1646 if (InVTNumElts % WidenNumElts == 0) { 1647 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, 1648 InOp, DAG.getIntPtrConstant(0)); 1649 // Extract the input and convert the shorten input vector. 1650 if (N->getNumOperands() == 1) 1651 return DAG.getNode(Opcode, DL, WidenVT, InVal); 1652 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1)); 1653 } 1654 } 1655 1656 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1657 SmallVector<SDValue, 16> Ops(WidenNumElts); 1658 EVT EltVT = WidenVT.getVectorElementType(); 1659 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1660 unsigned i; 1661 for (i=0; i < MinElts; ++i) { 1662 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 1663 DAG.getIntPtrConstant(i)); 1664 if (N->getNumOperands() == 1) 1665 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val); 1666 else 1667 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1)); 1668 } 1669 1670 SDValue UndefVal = DAG.getUNDEF(EltVT); 1671 for (; i < WidenNumElts; ++i) 1672 Ops[i] = UndefVal; 1673 1674 return DAG.getNode(ISD::BUILD_VECTOR, DL, WidenVT, &Ops[0], WidenNumElts); 1675 } 1676 1677 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) { 1678 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1679 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1680 SDValue ShOp = N->getOperand(1); 1681 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1682 } 1683 1684 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 1685 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1686 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1687 SDValue ShOp = N->getOperand(1); 1688 1689 EVT ShVT = ShOp.getValueType(); 1690 if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) { 1691 ShOp = GetWidenedVector(ShOp); 1692 ShVT = ShOp.getValueType(); 1693 } 1694 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), 1695 ShVT.getVectorElementType(), 1696 WidenVT.getVectorNumElements()); 1697 if (ShVT != ShWidenVT) 1698 ShOp = ModifyToType(ShOp, ShWidenVT); 1699 1700 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1701 } 1702 1703 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 1704 // Unary op widening. 1705 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1706 SDValue InOp = GetWidenedVector(N->getOperand(0)); 1707 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp); 1708 } 1709 1710 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 1711 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1712 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 1713 cast<VTSDNode>(N->getOperand(1))->getVT() 1714 .getVectorElementType(), 1715 WidenVT.getVectorNumElements()); 1716 SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 1717 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 1718 WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 1719 } 1720 1721 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { 1722 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); 1723 return GetWidenedVector(WidenVec); 1724 } 1725 1726 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { 1727 SDValue InOp = N->getOperand(0); 1728 EVT InVT = InOp.getValueType(); 1729 EVT VT = N->getValueType(0); 1730 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1731 DebugLoc dl = N->getDebugLoc(); 1732 1733 switch (getTypeAction(InVT)) { 1734 case TargetLowering::TypeLegal: 1735 break; 1736 case TargetLowering::TypePromoteInteger: 1737 // If the incoming type is a vector that is being promoted, then 1738 // we know that the elements are arranged differently and that we 1739 // must perform the conversion using a stack slot. 1740 if (InVT.isVector()) 1741 break; 1742 1743 // If the InOp is promoted to the same size, convert it. Otherwise, 1744 // fall out of the switch and widen the promoted input. 1745 InOp = GetPromotedInteger(InOp); 1746 InVT = InOp.getValueType(); 1747 if (WidenVT.bitsEq(InVT)) 1748 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 1749 break; 1750 case TargetLowering::TypeSoftenFloat: 1751 case TargetLowering::TypeExpandInteger: 1752 case TargetLowering::TypeExpandFloat: 1753 case TargetLowering::TypeScalarizeVector: 1754 case TargetLowering::TypeSplitVector: 1755 break; 1756 case TargetLowering::TypeWidenVector: 1757 // If the InOp is widened to the same size, convert it. Otherwise, fall 1758 // out of the switch and widen the widened input. 1759 InOp = GetWidenedVector(InOp); 1760 InVT = InOp.getValueType(); 1761 if (WidenVT.bitsEq(InVT)) 1762 // The input widens to the same size. Convert to the widen value. 1763 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 1764 break; 1765 } 1766 1767 unsigned WidenSize = WidenVT.getSizeInBits(); 1768 unsigned InSize = InVT.getSizeInBits(); 1769 // x86mmx is not an acceptable vector element type, so don't try. 1770 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) { 1771 // Determine new input vector type. The new input vector type will use 1772 // the same element type (if its a vector) or use the input type as a 1773 // vector. It is the same size as the type to widen to. 1774 EVT NewInVT; 1775 unsigned NewNumElts = WidenSize / InSize; 1776 if (InVT.isVector()) { 1777 EVT InEltVT = InVT.getVectorElementType(); 1778 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, 1779 WidenSize / InEltVT.getSizeInBits()); 1780 } else { 1781 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 1782 } 1783 1784 if (TLI.isTypeLegal(NewInVT)) { 1785 // Because the result and the input are different vector types, widening 1786 // the result could create a legal type but widening the input might make 1787 // it an illegal type that might lead to repeatedly splitting the input 1788 // and then widening it. To avoid this, we widen the input only if 1789 // it results in a legal type. 1790 SmallVector<SDValue, 16> Ops(NewNumElts); 1791 SDValue UndefVal = DAG.getUNDEF(InVT); 1792 Ops[0] = InOp; 1793 for (unsigned i = 1; i < NewNumElts; ++i) 1794 Ops[i] = UndefVal; 1795 1796 SDValue NewVec; 1797 if (InVT.isVector()) 1798 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1799 NewInVT, &Ops[0], NewNumElts); 1800 else 1801 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 1802 NewInVT, &Ops[0], NewNumElts); 1803 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec); 1804 } 1805 } 1806 1807 return CreateStackStoreLoad(InOp, WidenVT); 1808 } 1809 1810 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 1811 DebugLoc dl = N->getDebugLoc(); 1812 // Build a vector with undefined for the new nodes. 1813 EVT VT = N->getValueType(0); 1814 EVT EltVT = VT.getVectorElementType(); 1815 unsigned NumElts = VT.getVectorNumElements(); 1816 1817 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1818 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1819 1820 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 1821 NewOps.reserve(WidenNumElts); 1822 for (unsigned i = NumElts; i < WidenNumElts; ++i) 1823 NewOps.push_back(DAG.getUNDEF(EltVT)); 1824 1825 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size()); 1826 } 1827 1828 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 1829 EVT InVT = N->getOperand(0).getValueType(); 1830 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1831 DebugLoc dl = N->getDebugLoc(); 1832 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1833 unsigned NumInElts = InVT.getVectorNumElements(); 1834 unsigned NumOperands = N->getNumOperands(); 1835 1836 bool InputWidened = false; // Indicates we need to widen the input. 1837 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) { 1838 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 1839 // Add undef vectors to widen to correct length. 1840 unsigned NumConcat = WidenVT.getVectorNumElements() / 1841 InVT.getVectorNumElements(); 1842 SDValue UndefVal = DAG.getUNDEF(InVT); 1843 SmallVector<SDValue, 16> Ops(NumConcat); 1844 for (unsigned i=0; i < NumOperands; ++i) 1845 Ops[i] = N->getOperand(i); 1846 for (unsigned i = NumOperands; i != NumConcat; ++i) 1847 Ops[i] = UndefVal; 1848 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat); 1849 } 1850 } else { 1851 InputWidened = true; 1852 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 1853 // The inputs and the result are widen to the same value. 1854 unsigned i; 1855 for (i=1; i < NumOperands; ++i) 1856 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 1857 break; 1858 1859 if (i == NumOperands) 1860 // Everything but the first operand is an UNDEF so just return the 1861 // widened first operand. 1862 return GetWidenedVector(N->getOperand(0)); 1863 1864 if (NumOperands == 2) { 1865 // Replace concat of two operands with a shuffle. 1866 SmallVector<int, 16> MaskOps(WidenNumElts, -1); 1867 for (unsigned i = 0; i < NumInElts; ++i) { 1868 MaskOps[i] = i; 1869 MaskOps[i + NumInElts] = i + WidenNumElts; 1870 } 1871 return DAG.getVectorShuffle(WidenVT, dl, 1872 GetWidenedVector(N->getOperand(0)), 1873 GetWidenedVector(N->getOperand(1)), 1874 &MaskOps[0]); 1875 } 1876 } 1877 } 1878 1879 // Fall back to use extracts and build vector. 1880 EVT EltVT = WidenVT.getVectorElementType(); 1881 SmallVector<SDValue, 16> Ops(WidenNumElts); 1882 unsigned Idx = 0; 1883 for (unsigned i=0; i < NumOperands; ++i) { 1884 SDValue InOp = N->getOperand(i); 1885 if (InputWidened) 1886 InOp = GetWidenedVector(InOp); 1887 for (unsigned j=0; j < NumInElts; ++j) 1888 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1889 DAG.getIntPtrConstant(j)); 1890 } 1891 SDValue UndefVal = DAG.getUNDEF(EltVT); 1892 for (; Idx < WidenNumElts; ++Idx) 1893 Ops[Idx] = UndefVal; 1894 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1895 } 1896 1897 SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 1898 DebugLoc dl = N->getDebugLoc(); 1899 SDValue InOp = N->getOperand(0); 1900 SDValue RndOp = N->getOperand(3); 1901 SDValue SatOp = N->getOperand(4); 1902 1903 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1904 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1905 1906 EVT InVT = InOp.getValueType(); 1907 EVT InEltVT = InVT.getVectorElementType(); 1908 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1909 1910 SDValue DTyOp = DAG.getValueType(WidenVT); 1911 SDValue STyOp = DAG.getValueType(InWidenVT); 1912 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1913 1914 unsigned InVTNumElts = InVT.getVectorNumElements(); 1915 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 1916 InOp = GetWidenedVector(InOp); 1917 InVT = InOp.getValueType(); 1918 InVTNumElts = InVT.getVectorNumElements(); 1919 if (InVTNumElts == WidenNumElts) 1920 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1921 SatOp, CvtCode); 1922 } 1923 1924 if (TLI.isTypeLegal(InWidenVT)) { 1925 // Because the result and the input are different vector types, widening 1926 // the result could create a legal type but widening the input might make 1927 // it an illegal type that might lead to repeatedly splitting the input 1928 // and then widening it. To avoid this, we widen the input only if 1929 // it results in a legal type. 1930 if (WidenNumElts % InVTNumElts == 0) { 1931 // Widen the input and call convert on the widened input vector. 1932 unsigned NumConcat = WidenNumElts/InVTNumElts; 1933 SmallVector<SDValue, 16> Ops(NumConcat); 1934 Ops[0] = InOp; 1935 SDValue UndefVal = DAG.getUNDEF(InVT); 1936 for (unsigned i = 1; i != NumConcat; ++i) 1937 Ops[i] = UndefVal; 1938 1939 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat); 1940 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1941 SatOp, CvtCode); 1942 } 1943 1944 if (InVTNumElts % WidenNumElts == 0) { 1945 // Extract the input and convert the shorten input vector. 1946 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 1947 DAG.getIntPtrConstant(0)); 1948 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1949 SatOp, CvtCode); 1950 } 1951 } 1952 1953 // Otherwise unroll into some nasty scalar code and rebuild the vector. 1954 SmallVector<SDValue, 16> Ops(WidenNumElts); 1955 EVT EltVT = WidenVT.getVectorElementType(); 1956 DTyOp = DAG.getValueType(EltVT); 1957 STyOp = DAG.getValueType(InEltVT); 1958 1959 unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1960 unsigned i; 1961 for (i=0; i < MinElts; ++i) { 1962 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 1963 DAG.getIntPtrConstant(i)); 1964 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 1965 SatOp, CvtCode); 1966 } 1967 1968 SDValue UndefVal = DAG.getUNDEF(EltVT); 1969 for (; i < WidenNumElts; ++i) 1970 Ops[i] = UndefVal; 1971 1972 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1973 } 1974 1975 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 1976 EVT VT = N->getValueType(0); 1977 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1978 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1979 SDValue InOp = N->getOperand(0); 1980 SDValue Idx = N->getOperand(1); 1981 DebugLoc dl = N->getDebugLoc(); 1982 1983 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 1984 InOp = GetWidenedVector(InOp); 1985 1986 EVT InVT = InOp.getValueType(); 1987 1988 // Check if we can just return the input vector after widening. 1989 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1990 if (IdxVal == 0 && InVT == WidenVT) 1991 return InOp; 1992 1993 // Check if we can extract from the vector. 1994 unsigned InNumElts = InVT.getVectorNumElements(); 1995 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 1996 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 1997 1998 // We could try widening the input to the right length but for now, extract 1999 // the original elements, fill the rest with undefs and build a vector. 2000 SmallVector<SDValue, 16> Ops(WidenNumElts); 2001 EVT EltVT = VT.getVectorElementType(); 2002 unsigned NumElts = VT.getVectorNumElements(); 2003 unsigned i; 2004 for (i=0; i < NumElts; ++i) 2005 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2006 DAG.getIntPtrConstant(IdxVal+i)); 2007 2008 SDValue UndefVal = DAG.getUNDEF(EltVT); 2009 for (; i < WidenNumElts; ++i) 2010 Ops[i] = UndefVal; 2011 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 2012 } 2013 2014 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 2015 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2016 return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(), 2017 InOp.getValueType(), InOp, 2018 N->getOperand(1), N->getOperand(2)); 2019 } 2020 2021 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 2022 LoadSDNode *LD = cast<LoadSDNode>(N); 2023 ISD::LoadExtType ExtType = LD->getExtensionType(); 2024 2025 SDValue Result; 2026 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 2027 if (ExtType != ISD::NON_EXTLOAD) 2028 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); 2029 else 2030 Result = GenWidenVectorLoads(LdChain, LD); 2031 2032 // If we generate a single load, we can use that for the chain. Otherwise, 2033 // build a factor node to remember the multiple loads are independent and 2034 // chain to that. 2035 SDValue NewChain; 2036 if (LdChain.size() == 1) 2037 NewChain = LdChain[0]; 2038 else 2039 NewChain = DAG.getNode(ISD::TokenFactor, LD->getDebugLoc(), MVT::Other, 2040 &LdChain[0], LdChain.size()); 2041 2042 // Modified the chain - switch anything that used the old chain to use 2043 // the new one. 2044 ReplaceValueWith(SDValue(N, 1), NewChain); 2045 2046 return Result; 2047 } 2048 2049 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 2050 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2051 return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(), 2052 WidenVT, N->getOperand(0)); 2053 } 2054 2055 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 2056 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2057 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2058 2059 SDValue Cond1 = N->getOperand(0); 2060 EVT CondVT = Cond1.getValueType(); 2061 if (CondVT.isVector()) { 2062 EVT CondEltVT = CondVT.getVectorElementType(); 2063 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), 2064 CondEltVT, WidenNumElts); 2065 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector) 2066 Cond1 = GetWidenedVector(Cond1); 2067 2068 if (Cond1.getValueType() != CondWidenVT) 2069 Cond1 = ModifyToType(Cond1, CondWidenVT); 2070 } 2071 2072 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 2073 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 2074 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 2075 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 2076 WidenVT, Cond1, InOp1, InOp2); 2077 } 2078 2079 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 2080 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 2081 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 2082 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 2083 InOp1.getValueType(), N->getOperand(0), 2084 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 2085 } 2086 2087 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { 2088 assert(N->getValueType(0).isVector() == 2089 N->getOperand(0).getValueType().isVector() && 2090 "Scalar/Vector type mismatch"); 2091 if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N); 2092 2093 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2094 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2095 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2096 return DAG.getNode(ISD::SETCC, N->getDebugLoc(), WidenVT, 2097 InOp1, InOp2, N->getOperand(2)); 2098 } 2099 2100 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 2101 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2102 return DAG.getUNDEF(WidenVT); 2103 } 2104 2105 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 2106 EVT VT = N->getValueType(0); 2107 DebugLoc dl = N->getDebugLoc(); 2108 2109 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2110 unsigned NumElts = VT.getVectorNumElements(); 2111 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2112 2113 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2114 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2115 2116 // Adjust mask based on new input vector length. 2117 SmallVector<int, 16> NewMask; 2118 for (unsigned i = 0; i != NumElts; ++i) { 2119 int Idx = N->getMaskElt(i); 2120 if (Idx < (int)NumElts) 2121 NewMask.push_back(Idx); 2122 else 2123 NewMask.push_back(Idx - NumElts + WidenNumElts); 2124 } 2125 for (unsigned i = NumElts; i != WidenNumElts; ++i) 2126 NewMask.push_back(-1); 2127 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 2128 } 2129 2130 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 2131 assert(N->getValueType(0).isVector() && 2132 N->getOperand(0).getValueType().isVector() && 2133 "Operands must be vectors"); 2134 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2135 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2136 2137 SDValue InOp1 = N->getOperand(0); 2138 EVT InVT = InOp1.getValueType(); 2139 assert(InVT.isVector() && "can not widen non vector type"); 2140 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), 2141 InVT.getVectorElementType(), WidenNumElts); 2142 InOp1 = GetWidenedVector(InOp1); 2143 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2144 2145 // Assume that the input and output will be widen appropriately. If not, 2146 // we will have to unroll it at some point. 2147 assert(InOp1.getValueType() == WidenInVT && 2148 InOp2.getValueType() == WidenInVT && 2149 "Input not widened to expected type!"); 2150 (void)WidenInVT; 2151 return DAG.getNode(ISD::SETCC, N->getDebugLoc(), 2152 WidenVT, InOp1, InOp2, N->getOperand(2)); 2153 } 2154 2155 2156 //===----------------------------------------------------------------------===// 2157 // Widen Vector Operand 2158 //===----------------------------------------------------------------------===// 2159 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { 2160 DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; 2161 N->dump(&DAG); 2162 dbgs() << "\n"); 2163 SDValue Res = SDValue(); 2164 2165 // See if the target wants to custom widen this node. 2166 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2167 return false; 2168 2169 switch (N->getOpcode()) { 2170 default: 2171 #ifndef NDEBUG 2172 dbgs() << "WidenVectorOperand op #" << OpNo << ": "; 2173 N->dump(&DAG); 2174 dbgs() << "\n"; 2175 #endif 2176 llvm_unreachable("Do not know how to widen this operator's operand!"); 2177 2178 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break; 2179 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 2180 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 2181 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 2182 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 2183 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; 2184 2185 case ISD::FP_EXTEND: 2186 case ISD::FP_TO_SINT: 2187 case ISD::FP_TO_UINT: 2188 case ISD::SINT_TO_FP: 2189 case ISD::UINT_TO_FP: 2190 case ISD::TRUNCATE: 2191 case ISD::SIGN_EXTEND: 2192 case ISD::ZERO_EXTEND: 2193 case ISD::ANY_EXTEND: 2194 Res = WidenVecOp_Convert(N); 2195 break; 2196 } 2197 2198 // If Res is null, the sub-method took care of registering the result. 2199 if (!Res.getNode()) return false; 2200 2201 // If the result is N, the sub-method updated N in place. Tell the legalizer 2202 // core about this. 2203 if (Res.getNode() == N) 2204 return true; 2205 2206 2207 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2208 "Invalid operand expansion"); 2209 2210 ReplaceValueWith(SDValue(N, 0), Res); 2211 return false; 2212 } 2213 2214 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 2215 // Since the result is legal and the input is illegal, it is unlikely 2216 // that we can fix the input to a legal type so unroll the convert 2217 // into some scalar code and create a nasty build vector. 2218 EVT VT = N->getValueType(0); 2219 EVT EltVT = VT.getVectorElementType(); 2220 DebugLoc dl = N->getDebugLoc(); 2221 unsigned NumElts = VT.getVectorNumElements(); 2222 SDValue InOp = N->getOperand(0); 2223 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2224 InOp = GetWidenedVector(InOp); 2225 EVT InVT = InOp.getValueType(); 2226 EVT InEltVT = InVT.getVectorElementType(); 2227 2228 unsigned Opcode = N->getOpcode(); 2229 SmallVector<SDValue, 16> Ops(NumElts); 2230 for (unsigned i=0; i < NumElts; ++i) 2231 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 2232 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 2233 DAG.getIntPtrConstant(i))); 2234 2235 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 2236 } 2237 2238 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { 2239 EVT VT = N->getValueType(0); 2240 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2241 EVT InWidenVT = InOp.getValueType(); 2242 DebugLoc dl = N->getDebugLoc(); 2243 2244 // Check if we can convert between two legal vector types and extract. 2245 unsigned InWidenSize = InWidenVT.getSizeInBits(); 2246 unsigned Size = VT.getSizeInBits(); 2247 // x86mmx is not an acceptable vector element type, so don't try. 2248 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) { 2249 unsigned NewNumElts = InWidenSize / Size; 2250 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 2251 if (TLI.isTypeLegal(NewVT)) { 2252 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 2253 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 2254 DAG.getIntPtrConstant(0)); 2255 } 2256 } 2257 2258 return CreateStackStoreLoad(InOp, VT); 2259 } 2260 2261 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 2262 // If the input vector is not legal, it is likely that we will not find a 2263 // legal vector of the same size. Replace the concatenate vector with a 2264 // nasty build vector. 2265 EVT VT = N->getValueType(0); 2266 EVT EltVT = VT.getVectorElementType(); 2267 DebugLoc dl = N->getDebugLoc(); 2268 unsigned NumElts = VT.getVectorNumElements(); 2269 SmallVector<SDValue, 16> Ops(NumElts); 2270 2271 EVT InVT = N->getOperand(0).getValueType(); 2272 unsigned NumInElts = InVT.getVectorNumElements(); 2273 2274 unsigned Idx = 0; 2275 unsigned NumOperands = N->getNumOperands(); 2276 for (unsigned i=0; i < NumOperands; ++i) { 2277 SDValue InOp = N->getOperand(i); 2278 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2279 InOp = GetWidenedVector(InOp); 2280 for (unsigned j=0; j < NumInElts; ++j) 2281 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2282 DAG.getIntPtrConstant(j)); 2283 } 2284 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 2285 } 2286 2287 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 2288 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2289 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), 2290 N->getValueType(0), InOp, N->getOperand(1)); 2291 } 2292 2293 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 2294 SDValue InOp = GetWidenedVector(N->getOperand(0)); 2295 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 2296 N->getValueType(0), InOp, N->getOperand(1)); 2297 } 2298 2299 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 2300 // We have to widen the value but we want only to store the original 2301 // vector type. 2302 StoreSDNode *ST = cast<StoreSDNode>(N); 2303 2304 SmallVector<SDValue, 16> StChain; 2305 if (ST->isTruncatingStore()) 2306 GenWidenVectorTruncStores(StChain, ST); 2307 else 2308 GenWidenVectorStores(StChain, ST); 2309 2310 if (StChain.size() == 1) 2311 return StChain[0]; 2312 else 2313 return DAG.getNode(ISD::TokenFactor, ST->getDebugLoc(), 2314 MVT::Other,&StChain[0],StChain.size()); 2315 } 2316 2317 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { 2318 SDValue InOp0 = GetWidenedVector(N->getOperand(0)); 2319 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 2320 DebugLoc dl = N->getDebugLoc(); 2321 2322 // WARNING: In this code we widen the compare instruction with garbage. 2323 // This garbage may contain denormal floats which may be slow. Is this a real 2324 // concern ? Should we zero the unused lanes if this is a float compare ? 2325 2326 // Get a new SETCC node to compare the newly widened operands. 2327 // Only some of the compared elements are legal. 2328 EVT SVT = TLI.getSetCCResultType(InOp0.getValueType()); 2329 SDValue WideSETCC = DAG.getNode(ISD::SETCC, N->getDebugLoc(), 2330 SVT, InOp0, InOp1, N->getOperand(2)); 2331 2332 // Extract the needed results from the result vector. 2333 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), 2334 SVT.getVectorElementType(), 2335 N->getValueType(0).getVectorNumElements()); 2336 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, 2337 ResVT, WideSETCC, DAG.getIntPtrConstant(0)); 2338 2339 return PromoteTargetBoolean(CC, N->getValueType(0)); 2340 } 2341 2342 2343 //===----------------------------------------------------------------------===// 2344 // Vector Widening Utilities 2345 //===----------------------------------------------------------------------===// 2346 2347 // Utility function to find the type to chop up a widen vector for load/store 2348 // TLI: Target lowering used to determine legal types. 2349 // Width: Width left need to load/store. 2350 // WidenVT: The widen vector type to load to/store from 2351 // Align: If 0, don't allow use of a wider type 2352 // WidenEx: If Align is not 0, the amount additional we can load/store from. 2353 2354 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI, 2355 unsigned Width, EVT WidenVT, 2356 unsigned Align = 0, unsigned WidenEx = 0) { 2357 EVT WidenEltVT = WidenVT.getVectorElementType(); 2358 unsigned WidenWidth = WidenVT.getSizeInBits(); 2359 unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); 2360 unsigned AlignInBits = Align*8; 2361 2362 // If we have one element to load/store, return it. 2363 EVT RetVT = WidenEltVT; 2364 if (Width == WidenEltWidth) 2365 return RetVT; 2366 2367 // See if there is larger legal integer than the element type to load/store 2368 unsigned VT; 2369 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE; 2370 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) { 2371 EVT MemVT((MVT::SimpleValueType) VT); 2372 unsigned MemVTWidth = MemVT.getSizeInBits(); 2373 if (MemVT.getSizeInBits() <= WidenEltWidth) 2374 break; 2375 if (TLI.isTypeLegal(MemVT) && (WidenWidth % MemVTWidth) == 0 && 2376 isPowerOf2_32(WidenWidth / MemVTWidth) && 2377 (MemVTWidth <= Width || 2378 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2379 RetVT = MemVT; 2380 break; 2381 } 2382 } 2383 2384 // See if there is a larger vector type to load/store that has the same vector 2385 // element type and is evenly divisible with the WidenVT. 2386 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE; 2387 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) { 2388 EVT MemVT = (MVT::SimpleValueType) VT; 2389 unsigned MemVTWidth = MemVT.getSizeInBits(); 2390 if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() && 2391 (WidenWidth % MemVTWidth) == 0 && 2392 isPowerOf2_32(WidenWidth / MemVTWidth) && 2393 (MemVTWidth <= Width || 2394 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2395 if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT) 2396 return MemVT; 2397 } 2398 } 2399 2400 return RetVT; 2401 } 2402 2403 // Builds a vector type from scalar loads 2404 // VecTy: Resulting Vector type 2405 // LDOps: Load operators to build a vector type 2406 // [Start,End) the list of loads to use. 2407 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, 2408 SmallVector<SDValue, 16>& LdOps, 2409 unsigned Start, unsigned End) { 2410 DebugLoc dl = LdOps[Start].getDebugLoc(); 2411 EVT LdTy = LdOps[Start].getValueType(); 2412 unsigned Width = VecTy.getSizeInBits(); 2413 unsigned NumElts = Width / LdTy.getSizeInBits(); 2414 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts); 2415 2416 unsigned Idx = 1; 2417 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]); 2418 2419 for (unsigned i = Start + 1; i != End; ++i) { 2420 EVT NewLdTy = LdOps[i].getValueType(); 2421 if (NewLdTy != LdTy) { 2422 NumElts = Width / NewLdTy.getSizeInBits(); 2423 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts); 2424 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp); 2425 // Readjust position and vector position based on new load type 2426 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); 2427 LdTy = NewLdTy; 2428 } 2429 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i], 2430 DAG.getIntPtrConstant(Idx++)); 2431 } 2432 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp); 2433 } 2434 2435 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16> &LdChain, 2436 LoadSDNode *LD) { 2437 // The strategy assumes that we can efficiently load powers of two widths. 2438 // The routines chops the vector into the largest vector loads with the same 2439 // element type or scalar loads and then recombines it to the widen vector 2440 // type. 2441 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2442 unsigned WidenWidth = WidenVT.getSizeInBits(); 2443 EVT LdVT = LD->getMemoryVT(); 2444 DebugLoc dl = LD->getDebugLoc(); 2445 assert(LdVT.isVector() && WidenVT.isVector()); 2446 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 2447 2448 // Load information 2449 SDValue Chain = LD->getChain(); 2450 SDValue BasePtr = LD->getBasePtr(); 2451 unsigned Align = LD->getAlignment(); 2452 bool isVolatile = LD->isVolatile(); 2453 bool isNonTemporal = LD->isNonTemporal(); 2454 bool isInvariant = LD->isInvariant(); 2455 2456 int LdWidth = LdVT.getSizeInBits(); 2457 int WidthDiff = WidenWidth - LdWidth; // Difference 2458 unsigned LdAlign = (isVolatile) ? 0 : Align; // Allow wider loads 2459 2460 // Find the vector type that can load from. 2461 EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2462 int NewVTWidth = NewVT.getSizeInBits(); 2463 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(), 2464 isVolatile, isNonTemporal, isInvariant, Align); 2465 LdChain.push_back(LdOp.getValue(1)); 2466 2467 // Check if we can load the element with one instruction 2468 if (LdWidth <= NewVTWidth) { 2469 if (!NewVT.isVector()) { 2470 unsigned NumElts = WidenWidth / NewVTWidth; 2471 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2472 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2473 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp); 2474 } 2475 if (NewVT == WidenVT) 2476 return LdOp; 2477 2478 assert(WidenWidth % NewVTWidth == 0); 2479 unsigned NumConcat = WidenWidth / NewVTWidth; 2480 SmallVector<SDValue, 16> ConcatOps(NumConcat); 2481 SDValue UndefVal = DAG.getUNDEF(NewVT); 2482 ConcatOps[0] = LdOp; 2483 for (unsigned i = 1; i != NumConcat; ++i) 2484 ConcatOps[i] = UndefVal; 2485 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &ConcatOps[0], 2486 NumConcat); 2487 } 2488 2489 // Load vector by using multiple loads from largest vector to scalar 2490 SmallVector<SDValue, 16> LdOps; 2491 LdOps.push_back(LdOp); 2492 2493 LdWidth -= NewVTWidth; 2494 unsigned Offset = 0; 2495 2496 while (LdWidth > 0) { 2497 unsigned Increment = NewVTWidth / 8; 2498 Offset += Increment; 2499 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2500 DAG.getIntPtrConstant(Increment)); 2501 2502 SDValue L; 2503 if (LdWidth < NewVTWidth) { 2504 // Our current type we are using is too large, find a better size 2505 NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2506 NewVTWidth = NewVT.getSizeInBits(); 2507 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 2508 LD->getPointerInfo().getWithOffset(Offset), isVolatile, 2509 isNonTemporal, isInvariant, MinAlign(Align, Increment)); 2510 LdChain.push_back(L.getValue(1)); 2511 if (L->getValueType(0).isVector()) { 2512 SmallVector<SDValue, 16> Loads; 2513 Loads.push_back(L); 2514 unsigned size = L->getValueSizeInBits(0); 2515 while (size < LdOp->getValueSizeInBits(0)) { 2516 Loads.push_back(DAG.getUNDEF(L->getValueType(0))); 2517 size += L->getValueSizeInBits(0); 2518 } 2519 L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), 2520 &Loads[0], Loads.size()); 2521 } 2522 } else { 2523 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 2524 LD->getPointerInfo().getWithOffset(Offset), isVolatile, 2525 isNonTemporal, isInvariant, MinAlign(Align, Increment)); 2526 LdChain.push_back(L.getValue(1)); 2527 } 2528 2529 LdOps.push_back(L); 2530 2531 2532 LdWidth -= NewVTWidth; 2533 } 2534 2535 // Build the vector from the loads operations 2536 unsigned End = LdOps.size(); 2537 if (!LdOps[0].getValueType().isVector()) 2538 // All the loads are scalar loads. 2539 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End); 2540 2541 // If the load contains vectors, build the vector using concat vector. 2542 // All of the vectors used to loads are power of 2 and the scalars load 2543 // can be combined to make a power of 2 vector. 2544 SmallVector<SDValue, 16> ConcatOps(End); 2545 int i = End - 1; 2546 int Idx = End; 2547 EVT LdTy = LdOps[i].getValueType(); 2548 // First combine the scalar loads to a vector 2549 if (!LdTy.isVector()) { 2550 for (--i; i >= 0; --i) { 2551 LdTy = LdOps[i].getValueType(); 2552 if (LdTy.isVector()) 2553 break; 2554 } 2555 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i+1, End); 2556 } 2557 ConcatOps[--Idx] = LdOps[i]; 2558 for (--i; i >= 0; --i) { 2559 EVT NewLdTy = LdOps[i].getValueType(); 2560 if (NewLdTy != LdTy) { 2561 // Create a larger vector 2562 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy, 2563 &ConcatOps[Idx], End - Idx); 2564 Idx = End - 1; 2565 LdTy = NewLdTy; 2566 } 2567 ConcatOps[--Idx] = LdOps[i]; 2568 } 2569 2570 if (WidenWidth == LdTy.getSizeInBits()*(End - Idx)) 2571 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 2572 &ConcatOps[Idx], End - Idx); 2573 2574 // We need to fill the rest with undefs to build the vector 2575 unsigned NumOps = WidenWidth / LdTy.getSizeInBits(); 2576 SmallVector<SDValue, 16> WidenOps(NumOps); 2577 SDValue UndefVal = DAG.getUNDEF(LdTy); 2578 { 2579 unsigned i = 0; 2580 for (; i != End-Idx; ++i) 2581 WidenOps[i] = ConcatOps[Idx+i]; 2582 for (; i != NumOps; ++i) 2583 WidenOps[i] = UndefVal; 2584 } 2585 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &WidenOps[0],NumOps); 2586 } 2587 2588 SDValue 2589 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVector<SDValue, 16>& LdChain, 2590 LoadSDNode * LD, 2591 ISD::LoadExtType ExtType) { 2592 // For extension loads, it may not be more efficient to chop up the vector 2593 // and then extended it. Instead, we unroll the load and build a new vector. 2594 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2595 EVT LdVT = LD->getMemoryVT(); 2596 DebugLoc dl = LD->getDebugLoc(); 2597 assert(LdVT.isVector() && WidenVT.isVector()); 2598 2599 // Load information 2600 SDValue Chain = LD->getChain(); 2601 SDValue BasePtr = LD->getBasePtr(); 2602 unsigned Align = LD->getAlignment(); 2603 bool isVolatile = LD->isVolatile(); 2604 bool isNonTemporal = LD->isNonTemporal(); 2605 2606 EVT EltVT = WidenVT.getVectorElementType(); 2607 EVT LdEltVT = LdVT.getVectorElementType(); 2608 unsigned NumElts = LdVT.getVectorNumElements(); 2609 2610 // Load each element and widen 2611 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2612 SmallVector<SDValue, 16> Ops(WidenNumElts); 2613 unsigned Increment = LdEltVT.getSizeInBits() / 8; 2614 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, 2615 LD->getPointerInfo(), 2616 LdEltVT, isVolatile, isNonTemporal, Align); 2617 LdChain.push_back(Ops[0].getValue(1)); 2618 unsigned i = 0, Offset = Increment; 2619 for (i=1; i < NumElts; ++i, Offset += Increment) { 2620 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 2621 BasePtr, DAG.getIntPtrConstant(Offset)); 2622 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, 2623 LD->getPointerInfo().getWithOffset(Offset), LdEltVT, 2624 isVolatile, isNonTemporal, Align); 2625 LdChain.push_back(Ops[i].getValue(1)); 2626 } 2627 2628 // Fill the rest with undefs 2629 SDValue UndefVal = DAG.getUNDEF(EltVT); 2630 for (; i != WidenNumElts; ++i) 2631 Ops[i] = UndefVal; 2632 2633 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size()); 2634 } 2635 2636 2637 void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain, 2638 StoreSDNode *ST) { 2639 // The strategy assumes that we can efficiently store powers of two widths. 2640 // The routines chops the vector into the largest vector stores with the same 2641 // element type or scalar stores. 2642 SDValue Chain = ST->getChain(); 2643 SDValue BasePtr = ST->getBasePtr(); 2644 unsigned Align = ST->getAlignment(); 2645 bool isVolatile = ST->isVolatile(); 2646 bool isNonTemporal = ST->isNonTemporal(); 2647 SDValue ValOp = GetWidenedVector(ST->getValue()); 2648 DebugLoc dl = ST->getDebugLoc(); 2649 2650 EVT StVT = ST->getMemoryVT(); 2651 unsigned StWidth = StVT.getSizeInBits(); 2652 EVT ValVT = ValOp.getValueType(); 2653 unsigned ValWidth = ValVT.getSizeInBits(); 2654 EVT ValEltVT = ValVT.getVectorElementType(); 2655 unsigned ValEltWidth = ValEltVT.getSizeInBits(); 2656 assert(StVT.getVectorElementType() == ValEltVT); 2657 2658 int Idx = 0; // current index to store 2659 unsigned Offset = 0; // offset from base to store 2660 while (StWidth != 0) { 2661 // Find the largest vector type we can store with 2662 EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT); 2663 unsigned NewVTWidth = NewVT.getSizeInBits(); 2664 unsigned Increment = NewVTWidth / 8; 2665 if (NewVT.isVector()) { 2666 unsigned NumVTElts = NewVT.getVectorNumElements(); 2667 do { 2668 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp, 2669 DAG.getIntPtrConstant(Idx)); 2670 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 2671 ST->getPointerInfo().getWithOffset(Offset), 2672 isVolatile, isNonTemporal, 2673 MinAlign(Align, Offset))); 2674 StWidth -= NewVTWidth; 2675 Offset += Increment; 2676 Idx += NumVTElts; 2677 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2678 DAG.getIntPtrConstant(Increment)); 2679 } while (StWidth != 0 && StWidth >= NewVTWidth); 2680 } else { 2681 // Cast the vector to the scalar type we can store 2682 unsigned NumElts = ValWidth / NewVTWidth; 2683 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2684 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp); 2685 // Readjust index position based on new vector type 2686 Idx = Idx * ValEltWidth / NewVTWidth; 2687 do { 2688 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp, 2689 DAG.getIntPtrConstant(Idx++)); 2690 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 2691 ST->getPointerInfo().getWithOffset(Offset), 2692 isVolatile, isNonTemporal, 2693 MinAlign(Align, Offset))); 2694 StWidth -= NewVTWidth; 2695 Offset += Increment; 2696 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2697 DAG.getIntPtrConstant(Increment)); 2698 } while (StWidth != 0 && StWidth >= NewVTWidth); 2699 // Restore index back to be relative to the original widen element type 2700 Idx = Idx * NewVTWidth / ValEltWidth; 2701 } 2702 } 2703 } 2704 2705 void 2706 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVector<SDValue, 16>& StChain, 2707 StoreSDNode *ST) { 2708 // For extension loads, it may not be more efficient to truncate the vector 2709 // and then store it. Instead, we extract each element and then store it. 2710 SDValue Chain = ST->getChain(); 2711 SDValue BasePtr = ST->getBasePtr(); 2712 unsigned Align = ST->getAlignment(); 2713 bool isVolatile = ST->isVolatile(); 2714 bool isNonTemporal = ST->isNonTemporal(); 2715 SDValue ValOp = GetWidenedVector(ST->getValue()); 2716 DebugLoc dl = ST->getDebugLoc(); 2717 2718 EVT StVT = ST->getMemoryVT(); 2719 EVT ValVT = ValOp.getValueType(); 2720 2721 // It must be true that we the widen vector type is bigger than where 2722 // we need to store. 2723 assert(StVT.isVector() && ValOp.getValueType().isVector()); 2724 assert(StVT.bitsLT(ValOp.getValueType())); 2725 2726 // For truncating stores, we can not play the tricks of chopping legal 2727 // vector types and bit cast it to the right type. Instead, we unroll 2728 // the store. 2729 EVT StEltVT = StVT.getVectorElementType(); 2730 EVT ValEltVT = ValVT.getVectorElementType(); 2731 unsigned Increment = ValEltVT.getSizeInBits() / 8; 2732 unsigned NumElts = StVT.getVectorNumElements(); 2733 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 2734 DAG.getIntPtrConstant(0)); 2735 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, 2736 ST->getPointerInfo(), StEltVT, 2737 isVolatile, isNonTemporal, Align)); 2738 unsigned Offset = Increment; 2739 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 2740 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 2741 BasePtr, DAG.getIntPtrConstant(Offset)); 2742 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 2743 DAG.getIntPtrConstant(0)); 2744 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, 2745 ST->getPointerInfo().getWithOffset(Offset), 2746 StEltVT, isVolatile, isNonTemporal, 2747 MinAlign(Align, Offset))); 2748 } 2749 } 2750 2751 /// Modifies a vector input (widen or narrows) to a vector of NVT. The 2752 /// input vector must have the same element type as NVT. 2753 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) { 2754 // Note that InOp might have been widened so it might already have 2755 // the right width or it might need be narrowed. 2756 EVT InVT = InOp.getValueType(); 2757 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 2758 "input and widen element type must match"); 2759 DebugLoc dl = InOp.getDebugLoc(); 2760 2761 // Check if InOp already has the right width. 2762 if (InVT == NVT) 2763 return InOp; 2764 2765 unsigned InNumElts = InVT.getVectorNumElements(); 2766 unsigned WidenNumElts = NVT.getVectorNumElements(); 2767 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 2768 unsigned NumConcat = WidenNumElts / InNumElts; 2769 SmallVector<SDValue, 16> Ops(NumConcat); 2770 SDValue UndefVal = DAG.getUNDEF(InVT); 2771 Ops[0] = InOp; 2772 for (unsigned i = 1; i != NumConcat; ++i) 2773 Ops[i] = UndefVal; 2774 2775 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat); 2776 } 2777 2778 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 2779 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 2780 DAG.getIntPtrConstant(0)); 2781 2782 // Fall back to extract and build. 2783 SmallVector<SDValue, 16> Ops(WidenNumElts); 2784 EVT EltVT = NVT.getVectorElementType(); 2785 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 2786 unsigned Idx; 2787 for (Idx = 0; Idx < MinNumElts; ++Idx) 2788 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2789 DAG.getIntPtrConstant(Idx)); 2790 2791 SDValue UndefVal = DAG.getUNDEF(EltVT); 2792 for ( ; Idx < WidenNumElts; ++Idx) 2793 Ops[Idx] = UndefVal; 2794 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts); 2795 } 2796