1 //===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 #include "HexagonISelLowering.h" 11 #include "HexagonRegisterInfo.h" 12 #include "HexagonSubtarget.h" 13 #include "llvm/Support/CommandLine.h" 14 15 using namespace llvm; 16 17 static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 }; 18 static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 }; 19 static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 }; 20 static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 }; 21 22 23 void 24 HexagonTargetLowering::initializeHVXLowering() { 25 if (Subtarget.useHVX64BOps()) { 26 addRegisterClass(MVT::v64i8, &Hexagon::HvxVRRegClass); 27 addRegisterClass(MVT::v32i16, &Hexagon::HvxVRRegClass); 28 addRegisterClass(MVT::v16i32, &Hexagon::HvxVRRegClass); 29 addRegisterClass(MVT::v128i8, &Hexagon::HvxWRRegClass); 30 addRegisterClass(MVT::v64i16, &Hexagon::HvxWRRegClass); 31 addRegisterClass(MVT::v32i32, &Hexagon::HvxWRRegClass); 32 // These "short" boolean vector types should be legal because 33 // they will appear as results of vector compares. If they were 34 // not legal, type legalization would try to make them legal 35 // and that would require using operations that do not use or 36 // produce such types. That, in turn, would imply using custom 37 // nodes, which would be unoptimizable by the DAG combiner. 38 // The idea is to rely on target-independent operations as much 39 // as possible. 40 addRegisterClass(MVT::v16i1, &Hexagon::HvxQRRegClass); 41 addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass); 42 addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass); 43 addRegisterClass(MVT::v512i1, &Hexagon::HvxQRRegClass); 44 } else if (Subtarget.useHVX128BOps()) { 45 addRegisterClass(MVT::v128i8, &Hexagon::HvxVRRegClass); 46 addRegisterClass(MVT::v64i16, &Hexagon::HvxVRRegClass); 47 addRegisterClass(MVT::v32i32, &Hexagon::HvxVRRegClass); 48 addRegisterClass(MVT::v256i8, &Hexagon::HvxWRRegClass); 49 addRegisterClass(MVT::v128i16, &Hexagon::HvxWRRegClass); 50 addRegisterClass(MVT::v64i32, &Hexagon::HvxWRRegClass); 51 addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass); 52 addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass); 53 addRegisterClass(MVT::v128i1, &Hexagon::HvxQRRegClass); 54 addRegisterClass(MVT::v1024i1, &Hexagon::HvxQRRegClass); 55 } 56 57 // Set up operation actions. 58 59 bool Use64b = Subtarget.useHVX64BOps(); 60 ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128; 61 ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128; 62 MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8; 63 MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8; 64 65 auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) { 66 setOperationAction(Opc, FromTy, Promote); 67 AddPromotedToType(Opc, FromTy, ToTy); 68 }; 69 70 setOperationAction(ISD::VECTOR_SHUFFLE, ByteV, Legal); 71 setOperationAction(ISD::VECTOR_SHUFFLE, ByteW, Legal); 72 73 for (MVT T : LegalV) { 74 setIndexedLoadAction(ISD::POST_INC, T, Legal); 75 setIndexedStoreAction(ISD::POST_INC, T, Legal); 76 77 setOperationAction(ISD::AND, T, Legal); 78 setOperationAction(ISD::OR, T, Legal); 79 setOperationAction(ISD::XOR, T, Legal); 80 setOperationAction(ISD::ADD, T, Legal); 81 setOperationAction(ISD::SUB, T, Legal); 82 setOperationAction(ISD::CTPOP, T, Legal); 83 setOperationAction(ISD::CTLZ, T, Legal); 84 if (T != ByteV) { 85 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal); 86 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal); 87 setOperationAction(ISD::BSWAP, T, Legal); 88 } 89 90 setOperationAction(ISD::CTTZ, T, Custom); 91 setOperationAction(ISD::LOAD, T, Custom); 92 setOperationAction(ISD::MUL, T, Custom); 93 setOperationAction(ISD::MULHS, T, Custom); 94 setOperationAction(ISD::MULHU, T, Custom); 95 setOperationAction(ISD::BUILD_VECTOR, T, Custom); 96 // Make concat-vectors custom to handle concats of more than 2 vectors. 97 setOperationAction(ISD::CONCAT_VECTORS, T, Custom); 98 setOperationAction(ISD::INSERT_SUBVECTOR, T, Custom); 99 setOperationAction(ISD::INSERT_VECTOR_ELT, T, Custom); 100 setOperationAction(ISD::EXTRACT_SUBVECTOR, T, Custom); 101 setOperationAction(ISD::EXTRACT_VECTOR_ELT, T, Custom); 102 setOperationAction(ISD::ANY_EXTEND, T, Custom); 103 setOperationAction(ISD::SIGN_EXTEND, T, Custom); 104 setOperationAction(ISD::ZERO_EXTEND, T, Custom); 105 if (T != ByteV) { 106 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom); 107 // HVX only has shifts of words and halfwords. 108 setOperationAction(ISD::SRA, T, Custom); 109 setOperationAction(ISD::SHL, T, Custom); 110 setOperationAction(ISD::SRL, T, Custom); 111 112 // Promote all shuffles to operate on vectors of bytes. 113 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV); 114 } 115 116 setCondCodeAction(ISD::SETNE, T, Expand); 117 setCondCodeAction(ISD::SETLE, T, Expand); 118 setCondCodeAction(ISD::SETGE, T, Expand); 119 setCondCodeAction(ISD::SETLT, T, Expand); 120 setCondCodeAction(ISD::SETULE, T, Expand); 121 setCondCodeAction(ISD::SETUGE, T, Expand); 122 setCondCodeAction(ISD::SETULT, T, Expand); 123 } 124 125 for (MVT T : LegalW) { 126 // Custom-lower BUILD_VECTOR for vector pairs. The standard (target- 127 // independent) handling of it would convert it to a load, which is 128 // not always the optimal choice. 129 setOperationAction(ISD::BUILD_VECTOR, T, Custom); 130 // Make concat-vectors custom to handle concats of more than 2 vectors. 131 setOperationAction(ISD::CONCAT_VECTORS, T, Custom); 132 133 // Custom-lower these operations for pairs. Expand them into a concat 134 // of the corresponding operations on individual vectors. 135 setOperationAction(ISD::ANY_EXTEND, T, Custom); 136 setOperationAction(ISD::SIGN_EXTEND, T, Custom); 137 setOperationAction(ISD::ZERO_EXTEND, T, Custom); 138 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Custom); 139 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom); 140 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal); 141 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal); 142 143 setOperationAction(ISD::LOAD, T, Custom); 144 setOperationAction(ISD::STORE, T, Custom); 145 setOperationAction(ISD::CTLZ, T, Custom); 146 setOperationAction(ISD::CTTZ, T, Custom); 147 setOperationAction(ISD::CTPOP, T, Custom); 148 149 setOperationAction(ISD::ADD, T, Legal); 150 setOperationAction(ISD::SUB, T, Legal); 151 setOperationAction(ISD::MUL, T, Custom); 152 setOperationAction(ISD::MULHS, T, Custom); 153 setOperationAction(ISD::MULHU, T, Custom); 154 setOperationAction(ISD::AND, T, Custom); 155 setOperationAction(ISD::OR, T, Custom); 156 setOperationAction(ISD::XOR, T, Custom); 157 setOperationAction(ISD::SETCC, T, Custom); 158 setOperationAction(ISD::VSELECT, T, Custom); 159 if (T != ByteW) { 160 setOperationAction(ISD::SRA, T, Custom); 161 setOperationAction(ISD::SHL, T, Custom); 162 setOperationAction(ISD::SRL, T, Custom); 163 164 // Promote all shuffles to operate on vectors of bytes. 165 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW); 166 } 167 } 168 169 // Boolean vectors. 170 171 for (MVT T : LegalW) { 172 // Boolean types for vector pairs will overlap with the boolean 173 // types for single vectors, e.g. 174 // v64i8 -> v64i1 (single) 175 // v64i16 -> v64i1 (pair) 176 // Set these actions first, and allow the single actions to overwrite 177 // any duplicates. 178 MVT BoolW = MVT::getVectorVT(MVT::i1, T.getVectorNumElements()); 179 setOperationAction(ISD::SETCC, BoolW, Custom); 180 setOperationAction(ISD::AND, BoolW, Custom); 181 setOperationAction(ISD::OR, BoolW, Custom); 182 setOperationAction(ISD::XOR, BoolW, Custom); 183 } 184 185 for (MVT T : LegalV) { 186 MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements()); 187 setOperationAction(ISD::BUILD_VECTOR, BoolV, Custom); 188 setOperationAction(ISD::CONCAT_VECTORS, BoolV, Custom); 189 setOperationAction(ISD::INSERT_SUBVECTOR, BoolV, Custom); 190 setOperationAction(ISD::INSERT_VECTOR_ELT, BoolV, Custom); 191 setOperationAction(ISD::EXTRACT_SUBVECTOR, BoolV, Custom); 192 setOperationAction(ISD::EXTRACT_VECTOR_ELT, BoolV, Custom); 193 setOperationAction(ISD::AND, BoolV, Legal); 194 setOperationAction(ISD::OR, BoolV, Legal); 195 setOperationAction(ISD::XOR, BoolV, Legal); 196 } 197 } 198 199 SDValue 200 HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops, 201 const SDLoc &dl, SelectionDAG &DAG) const { 202 SmallVector<SDValue,4> IntOps; 203 IntOps.push_back(DAG.getConstant(IntId, dl, MVT::i32)); 204 for (const SDValue &Op : Ops) 205 IntOps.push_back(Op); 206 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResTy, IntOps); 207 } 208 209 MVT 210 HexagonTargetLowering::typeJoin(const TypePair &Tys) const { 211 assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType()); 212 213 MVT ElemTy = Tys.first.getVectorElementType(); 214 return MVT::getVectorVT(ElemTy, Tys.first.getVectorNumElements() + 215 Tys.second.getVectorNumElements()); 216 } 217 218 HexagonTargetLowering::TypePair 219 HexagonTargetLowering::typeSplit(MVT VecTy) const { 220 assert(VecTy.isVector()); 221 unsigned NumElem = VecTy.getVectorNumElements(); 222 assert((NumElem % 2) == 0 && "Expecting even-sized vector type"); 223 MVT HalfTy = MVT::getVectorVT(VecTy.getVectorElementType(), NumElem/2); 224 return { HalfTy, HalfTy }; 225 } 226 227 MVT 228 HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const { 229 MVT ElemTy = VecTy.getVectorElementType(); 230 MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() * Factor); 231 return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements()); 232 } 233 234 MVT 235 HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const { 236 MVT ElemTy = VecTy.getVectorElementType(); 237 MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() / Factor); 238 return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements()); 239 } 240 241 SDValue 242 HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy, 243 SelectionDAG &DAG) const { 244 if (ty(Vec).getVectorElementType() == ElemTy) 245 return Vec; 246 MVT CastTy = tyVector(Vec.getValueType().getSimpleVT(), ElemTy); 247 return DAG.getBitcast(CastTy, Vec); 248 } 249 250 SDValue 251 HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl, 252 SelectionDAG &DAG) const { 253 return DAG.getNode(ISD::CONCAT_VECTORS, dl, typeJoin(ty(Ops)), 254 Ops.second, Ops.first); 255 } 256 257 HexagonTargetLowering::VectorPair 258 HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl, 259 SelectionDAG &DAG) const { 260 TypePair Tys = typeSplit(ty(Vec)); 261 if (Vec.getOpcode() == HexagonISD::QCAT) 262 return VectorPair(Vec.getOperand(0), Vec.getOperand(1)); 263 return DAG.SplitVector(Vec, dl, Tys.first, Tys.second); 264 } 265 266 bool 267 HexagonTargetLowering::isHvxSingleTy(MVT Ty) const { 268 return Subtarget.isHVXVectorType(Ty) && 269 Ty.getSizeInBits() == 8 * Subtarget.getVectorLength(); 270 } 271 272 bool 273 HexagonTargetLowering::isHvxPairTy(MVT Ty) const { 274 return Subtarget.isHVXVectorType(Ty) && 275 Ty.getSizeInBits() == 16 * Subtarget.getVectorLength(); 276 } 277 278 SDValue 279 HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy, 280 SelectionDAG &DAG) const { 281 if (ElemIdx.getValueType().getSimpleVT() != MVT::i32) 282 ElemIdx = DAG.getBitcast(MVT::i32, ElemIdx); 283 284 unsigned ElemWidth = ElemTy.getSizeInBits(); 285 if (ElemWidth == 8) 286 return ElemIdx; 287 288 unsigned L = Log2_32(ElemWidth/8); 289 const SDLoc &dl(ElemIdx); 290 return DAG.getNode(ISD::SHL, dl, MVT::i32, 291 {ElemIdx, DAG.getConstant(L, dl, MVT::i32)}); 292 } 293 294 SDValue 295 HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy, 296 SelectionDAG &DAG) const { 297 unsigned ElemWidth = ElemTy.getSizeInBits(); 298 assert(ElemWidth >= 8 && ElemWidth <= 32); 299 if (ElemWidth == 32) 300 return Idx; 301 302 if (ty(Idx) != MVT::i32) 303 Idx = DAG.getBitcast(MVT::i32, Idx); 304 const SDLoc &dl(Idx); 305 SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32); 306 SDValue SubIdx = DAG.getNode(ISD::AND, dl, MVT::i32, {Idx, Mask}); 307 return SubIdx; 308 } 309 310 SDValue 311 HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0, 312 SDValue Op1, ArrayRef<int> Mask, 313 SelectionDAG &DAG) const { 314 MVT OpTy = ty(Op0); 315 assert(OpTy == ty(Op1)); 316 317 MVT ElemTy = OpTy.getVectorElementType(); 318 if (ElemTy == MVT::i8) 319 return DAG.getVectorShuffle(OpTy, dl, Op0, Op1, Mask); 320 assert(ElemTy.getSizeInBits() >= 8); 321 322 MVT ResTy = tyVector(OpTy, MVT::i8); 323 unsigned ElemSize = ElemTy.getSizeInBits() / 8; 324 325 SmallVector<int,128> ByteMask; 326 for (int M : Mask) { 327 if (M < 0) { 328 for (unsigned I = 0; I != ElemSize; ++I) 329 ByteMask.push_back(-1); 330 } else { 331 int NewM = M*ElemSize; 332 for (unsigned I = 0; I != ElemSize; ++I) 333 ByteMask.push_back(NewM+I); 334 } 335 } 336 assert(ResTy.getVectorNumElements() == ByteMask.size()); 337 return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG), 338 opCastElem(Op1, MVT::i8, DAG), ByteMask); 339 } 340 341 SDValue 342 HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values, 343 const SDLoc &dl, MVT VecTy, 344 SelectionDAG &DAG) const { 345 unsigned VecLen = Values.size(); 346 MachineFunction &MF = DAG.getMachineFunction(); 347 MVT ElemTy = VecTy.getVectorElementType(); 348 unsigned ElemWidth = ElemTy.getSizeInBits(); 349 unsigned HwLen = Subtarget.getVectorLength(); 350 351 unsigned ElemSize = ElemWidth / 8; 352 assert(ElemSize*VecLen == HwLen); 353 SmallVector<SDValue,32> Words; 354 355 if (VecTy.getVectorElementType() != MVT::i32) { 356 assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size"); 357 unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2; 358 MVT PartVT = MVT::getVectorVT(VecTy.getVectorElementType(), OpsPerWord); 359 for (unsigned i = 0; i != VecLen; i += OpsPerWord) { 360 SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG); 361 Words.push_back(DAG.getBitcast(MVT::i32, W)); 362 } 363 } else { 364 Words.assign(Values.begin(), Values.end()); 365 } 366 367 unsigned NumWords = Words.size(); 368 bool IsSplat = true, IsUndef = true; 369 SDValue SplatV; 370 for (unsigned i = 0; i != NumWords && IsSplat; ++i) { 371 if (isUndef(Words[i])) 372 continue; 373 IsUndef = false; 374 if (!SplatV.getNode()) 375 SplatV = Words[i]; 376 else if (SplatV != Words[i]) 377 IsSplat = false; 378 } 379 if (IsUndef) 380 return DAG.getUNDEF(VecTy); 381 if (IsSplat) { 382 assert(SplatV.getNode()); 383 auto *IdxN = dyn_cast<ConstantSDNode>(SplatV.getNode()); 384 if (IdxN && IdxN->isNullValue()) 385 return getZero(dl, VecTy, DAG); 386 return DAG.getNode(HexagonISD::VSPLATW, dl, VecTy, SplatV); 387 } 388 389 // Delay recognizing constant vectors until here, so that we can generate 390 // a vsplat. 391 SmallVector<ConstantInt*, 128> Consts(VecLen); 392 bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts); 393 if (AllConst) { 394 ArrayRef<Constant*> Tmp((Constant**)Consts.begin(), 395 (Constant**)Consts.end()); 396 Constant *CV = ConstantVector::get(Tmp); 397 unsigned Align = HwLen; 398 SDValue CP = LowerConstantPool(DAG.getConstantPool(CV, VecTy, Align), DAG); 399 return DAG.getLoad(VecTy, dl, DAG.getEntryNode(), CP, 400 MachinePointerInfo::getConstantPool(MF), Align); 401 } 402 403 // Construct two halves in parallel, then or them together. 404 assert(4*Words.size() == Subtarget.getVectorLength()); 405 SDValue HalfV0 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG); 406 SDValue HalfV1 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG); 407 SDValue S = DAG.getConstant(4, dl, MVT::i32); 408 for (unsigned i = 0; i != NumWords/2; ++i) { 409 SDValue N = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, 410 {HalfV0, Words[i]}); 411 SDValue M = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, 412 {HalfV1, Words[i+NumWords/2]}); 413 HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, S}); 414 HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, S}); 415 } 416 417 HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, 418 {HalfV0, DAG.getConstant(HwLen/2, dl, MVT::i32)}); 419 SDValue DstV = DAG.getNode(ISD::OR, dl, VecTy, {HalfV0, HalfV1}); 420 return DstV; 421 } 422 423 SDValue 424 HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl, 425 unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const { 426 MVT PredTy = ty(PredV); 427 unsigned HwLen = Subtarget.getVectorLength(); 428 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 429 430 if (Subtarget.isHVXVectorType(PredTy, true)) { 431 // Move the vector predicate SubV to a vector register, and scale it 432 // down to match the representation (bytes per type element) that VecV 433 // uses. The scaling down will pick every 2nd or 4th (every Scale-th 434 // in general) element and put them at the front of the resulting 435 // vector. This subvector will then be inserted into the Q2V of VecV. 436 // To avoid having an operation that generates an illegal type (short 437 // vector), generate a full size vector. 438 // 439 SDValue T = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, PredV); 440 SmallVector<int,128> Mask(HwLen); 441 // Scale = BitBytes(PredV) / Given BitBytes. 442 unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes); 443 unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes; 444 445 for (unsigned i = 0; i != HwLen; ++i) { 446 unsigned Num = i % Scale; 447 unsigned Off = i / Scale; 448 Mask[BlockLen*Num + Off] = i; 449 } 450 SDValue S = DAG.getVectorShuffle(ByteTy, dl, T, DAG.getUNDEF(ByteTy), Mask); 451 if (!ZeroFill) 452 return S; 453 // Fill the bytes beyond BlockLen with 0s. 454 MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); 455 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy, 456 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG); 457 SDValue M = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Q); 458 return DAG.getNode(ISD::AND, dl, ByteTy, S, M); 459 } 460 461 // Make sure that this is a valid scalar predicate. 462 assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1); 463 464 unsigned Bytes = 8 / PredTy.getVectorNumElements(); 465 SmallVector<SDValue,4> Words[2]; 466 unsigned IdxW = 0; 467 468 auto Lo32 = [&DAG, &dl] (SDValue P) { 469 return DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, P); 470 }; 471 auto Hi32 = [&DAG, &dl] (SDValue P) { 472 return DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, P); 473 }; 474 475 SDValue W0 = isUndef(PredV) 476 ? DAG.getUNDEF(MVT::i64) 477 : DAG.getNode(HexagonISD::P2D, dl, MVT::i64, PredV); 478 Words[IdxW].push_back(Hi32(W0)); 479 Words[IdxW].push_back(Lo32(W0)); 480 481 while (Bytes < BitBytes) { 482 IdxW ^= 1; 483 Words[IdxW].clear(); 484 485 if (Bytes < 4) { 486 for (const SDValue &W : Words[IdxW ^ 1]) { 487 SDValue T = expandPredicate(W, dl, DAG); 488 Words[IdxW].push_back(Hi32(T)); 489 Words[IdxW].push_back(Lo32(T)); 490 } 491 } else { 492 for (const SDValue &W : Words[IdxW ^ 1]) { 493 Words[IdxW].push_back(W); 494 Words[IdxW].push_back(W); 495 } 496 } 497 Bytes *= 2; 498 } 499 500 assert(Bytes == BitBytes); 501 502 SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy); 503 SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32); 504 for (const SDValue &W : Words[IdxW]) { 505 Vec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Vec, S4); 506 Vec = DAG.getNode(HexagonISD::VINSERTW0, dl, ByteTy, Vec, W); 507 } 508 509 return Vec; 510 } 511 512 SDValue 513 HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values, 514 const SDLoc &dl, MVT VecTy, 515 SelectionDAG &DAG) const { 516 // Construct a vector V of bytes, such that a comparison V >u 0 would 517 // produce the required vector predicate. 518 unsigned VecLen = Values.size(); 519 unsigned HwLen = Subtarget.getVectorLength(); 520 assert(VecLen <= HwLen || VecLen == 8*HwLen); 521 SmallVector<SDValue,128> Bytes; 522 bool AllT = true, AllF = true; 523 524 auto IsTrue = [] (SDValue V) { 525 if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode())) 526 return !N->isNullValue(); 527 return false; 528 }; 529 auto IsFalse = [] (SDValue V) { 530 if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode())) 531 return N->isNullValue(); 532 return false; 533 }; 534 535 if (VecLen <= HwLen) { 536 // In the hardware, each bit of a vector predicate corresponds to a byte 537 // of a vector register. Calculate how many bytes does a bit of VecTy 538 // correspond to. 539 assert(HwLen % VecLen == 0); 540 unsigned BitBytes = HwLen / VecLen; 541 for (SDValue V : Values) { 542 AllT &= IsTrue(V); 543 AllF &= IsFalse(V); 544 545 SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(V, dl, MVT::i8) 546 : DAG.getUNDEF(MVT::i8); 547 for (unsigned B = 0; B != BitBytes; ++B) 548 Bytes.push_back(Ext); 549 } 550 } else { 551 // There are as many i1 values, as there are bits in a vector register. 552 // Divide the values into groups of 8 and check that each group consists 553 // of the same value (ignoring undefs). 554 for (unsigned I = 0; I != VecLen; I += 8) { 555 unsigned B = 0; 556 // Find the first non-undef value in this group. 557 for (; B != 8; ++B) { 558 if (!Values[I+B].isUndef()) 559 break; 560 } 561 SDValue F = Values[I+B]; 562 AllT &= IsTrue(F); 563 AllF &= IsFalse(F); 564 565 SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(F, dl, MVT::i8) 566 : DAG.getUNDEF(MVT::i8); 567 Bytes.push_back(Ext); 568 // Verify that the rest of values in the group are the same as the 569 // first. 570 for (; B != 8; ++B) 571 assert(Values[I+B].isUndef() || Values[I+B] == F); 572 } 573 } 574 575 if (AllT) 576 return DAG.getNode(HexagonISD::QTRUE, dl, VecTy); 577 if (AllF) 578 return DAG.getNode(HexagonISD::QFALSE, dl, VecTy); 579 580 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 581 SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG); 582 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec); 583 } 584 585 SDValue 586 HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV, 587 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const { 588 MVT ElemTy = ty(VecV).getVectorElementType(); 589 590 unsigned ElemWidth = ElemTy.getSizeInBits(); 591 assert(ElemWidth >= 8 && ElemWidth <= 32); 592 (void)ElemWidth; 593 594 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG); 595 SDValue ExWord = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, 596 {VecV, ByteIdx}); 597 if (ElemTy == MVT::i32) 598 return ExWord; 599 600 // Have an extracted word, need to extract the smaller element out of it. 601 // 1. Extract the bits of (the original) IdxV that correspond to the index 602 // of the desired element in the 32-bit word. 603 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG); 604 // 2. Extract the element from the word. 605 SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord); 606 return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG); 607 } 608 609 SDValue 610 HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV, 611 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const { 612 // Implement other return types if necessary. 613 assert(ResTy == MVT::i1); 614 615 unsigned HwLen = Subtarget.getVectorLength(); 616 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 617 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV); 618 619 unsigned Scale = HwLen / ty(VecV).getVectorNumElements(); 620 SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32); 621 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV); 622 623 SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG); 624 SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32); 625 return getInstr(Hexagon::C2_cmpgtui, dl, MVT::i1, {ExtB, Zero}, DAG); 626 } 627 628 SDValue 629 HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV, 630 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const { 631 MVT ElemTy = ty(VecV).getVectorElementType(); 632 633 unsigned ElemWidth = ElemTy.getSizeInBits(); 634 assert(ElemWidth >= 8 && ElemWidth <= 32); 635 (void)ElemWidth; 636 637 auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV, 638 SDValue ByteIdxV) { 639 MVT VecTy = ty(VecV); 640 unsigned HwLen = Subtarget.getVectorLength(); 641 SDValue MaskV = DAG.getNode(ISD::AND, dl, MVT::i32, 642 {ByteIdxV, DAG.getConstant(-4, dl, MVT::i32)}); 643 SDValue RotV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {VecV, MaskV}); 644 SDValue InsV = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, {RotV, ValV}); 645 SDValue SubV = DAG.getNode(ISD::SUB, dl, MVT::i32, 646 {DAG.getConstant(HwLen, dl, MVT::i32), MaskV}); 647 SDValue TorV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {InsV, SubV}); 648 return TorV; 649 }; 650 651 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG); 652 if (ElemTy == MVT::i32) 653 return InsertWord(VecV, ValV, ByteIdx); 654 655 // If this is not inserting a 32-bit word, convert it into such a thing. 656 // 1. Extract the existing word from the target vector. 657 SDValue WordIdx = DAG.getNode(ISD::SRL, dl, MVT::i32, 658 {ByteIdx, DAG.getConstant(2, dl, MVT::i32)}); 659 SDValue Ext = extractHvxElementReg(opCastElem(VecV, MVT::i32, DAG), WordIdx, 660 dl, MVT::i32, DAG); 661 662 // 2. Treating the extracted word as a 32-bit vector, insert the given 663 // value into it. 664 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG); 665 MVT SubVecTy = tyVector(ty(Ext), ElemTy); 666 SDValue Ins = insertVector(DAG.getBitcast(SubVecTy, Ext), 667 ValV, SubIdx, dl, ElemTy, DAG); 668 669 // 3. Insert the 32-bit word back into the original vector. 670 return InsertWord(VecV, Ins, ByteIdx); 671 } 672 673 SDValue 674 HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV, 675 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const { 676 unsigned HwLen = Subtarget.getVectorLength(); 677 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 678 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV); 679 680 unsigned Scale = HwLen / ty(VecV).getVectorNumElements(); 681 SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32); 682 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV); 683 ValV = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, ValV); 684 685 SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG); 686 return DAG.getNode(HexagonISD::V2Q, dl, ty(VecV), InsV); 687 } 688 689 SDValue 690 HexagonTargetLowering::extractHvxSubvectorReg(SDValue VecV, SDValue IdxV, 691 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const { 692 MVT VecTy = ty(VecV); 693 unsigned HwLen = Subtarget.getVectorLength(); 694 unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue(); 695 MVT ElemTy = VecTy.getVectorElementType(); 696 unsigned ElemWidth = ElemTy.getSizeInBits(); 697 698 // If the source vector is a vector pair, get the single vector containing 699 // the subvector of interest. The subvector will never overlap two single 700 // vectors. 701 if (isHvxPairTy(VecTy)) { 702 unsigned SubIdx; 703 if (Idx * ElemWidth >= 8*HwLen) { 704 SubIdx = Hexagon::vsub_hi; 705 Idx -= VecTy.getVectorNumElements() / 2; 706 } else { 707 SubIdx = Hexagon::vsub_lo; 708 } 709 VecTy = typeSplit(VecTy).first; 710 VecV = DAG.getTargetExtractSubreg(SubIdx, dl, VecTy, VecV); 711 if (VecTy == ResTy) 712 return VecV; 713 } 714 715 // The only meaningful subvectors of a single HVX vector are those that 716 // fit in a scalar register. 717 assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64); 718 719 MVT WordTy = tyVector(VecTy, MVT::i32); 720 SDValue WordVec = DAG.getBitcast(WordTy, VecV); 721 unsigned WordIdx = (Idx*ElemWidth) / 32; 722 723 SDValue W0Idx = DAG.getConstant(WordIdx, dl, MVT::i32); 724 SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG); 725 if (ResTy.getSizeInBits() == 32) 726 return DAG.getBitcast(ResTy, W0); 727 728 SDValue W1Idx = DAG.getConstant(WordIdx+1, dl, MVT::i32); 729 SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG); 730 SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, {W1, W0}); 731 return DAG.getBitcast(ResTy, WW); 732 } 733 734 SDValue 735 HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV, 736 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const { 737 MVT VecTy = ty(VecV); 738 unsigned HwLen = Subtarget.getVectorLength(); 739 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 740 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV); 741 // IdxV is required to be a constant. 742 unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue(); 743 744 unsigned ResLen = ResTy.getVectorNumElements(); 745 unsigned BitBytes = HwLen / VecTy.getVectorNumElements(); 746 unsigned Offset = Idx * BitBytes; 747 SDValue Undef = DAG.getUNDEF(ByteTy); 748 SmallVector<int,128> Mask; 749 750 if (Subtarget.isHVXVectorType(ResTy, true)) { 751 // Converting between two vector predicates. Since the result is shorter 752 // than the source, it will correspond to a vector predicate with the 753 // relevant bits replicated. The replication count is the ratio of the 754 // source and target vector lengths. 755 unsigned Rep = VecTy.getVectorNumElements() / ResLen; 756 assert(isPowerOf2_32(Rep) && HwLen % Rep == 0); 757 for (unsigned i = 0; i != HwLen/Rep; ++i) { 758 for (unsigned j = 0; j != Rep; ++j) 759 Mask.push_back(i + Offset); 760 } 761 SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask); 762 return DAG.getNode(HexagonISD::V2Q, dl, ResTy, ShuffV); 763 } 764 765 // Converting between a vector predicate and a scalar predicate. In the 766 // vector predicate, a group of BitBytes bits will correspond to a single 767 // i1 element of the source vector type. Those bits will all have the same 768 // value. The same will be true for ByteVec, where each byte corresponds 769 // to a bit in the vector predicate. 770 // The algorithm is to traverse the ByteVec, going over the i1 values from 771 // the source vector, and generate the corresponding representation in an 772 // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the 773 // elements so that the interesting 8 bytes will be in the low end of the 774 // vector. 775 unsigned Rep = 8 / ResLen; 776 // Make sure the output fill the entire vector register, so repeat the 777 // 8-byte groups as many times as necessary. 778 for (unsigned r = 0; r != HwLen/ResLen; ++r) { 779 // This will generate the indexes of the 8 interesting bytes. 780 for (unsigned i = 0; i != ResLen; ++i) { 781 for (unsigned j = 0; j != Rep; ++j) 782 Mask.push_back(Offset + i*BitBytes); 783 } 784 } 785 786 SDValue Zero = getZero(dl, MVT::i32, DAG); 787 SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask); 788 // Combine the two low words from ShuffV into a v8i8, and byte-compare 789 // them against 0. 790 SDValue W0 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, {ShuffV, Zero}); 791 SDValue W1 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, 792 {ShuffV, DAG.getConstant(4, dl, MVT::i32)}); 793 SDValue Vec64 = DAG.getNode(HexagonISD::COMBINE, dl, MVT::v8i8, {W1, W0}); 794 return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy, 795 {Vec64, DAG.getTargetConstant(0, dl, MVT::i32)}, DAG); 796 } 797 798 SDValue 799 HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV, 800 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const { 801 MVT VecTy = ty(VecV); 802 MVT SubTy = ty(SubV); 803 unsigned HwLen = Subtarget.getVectorLength(); 804 MVT ElemTy = VecTy.getVectorElementType(); 805 unsigned ElemWidth = ElemTy.getSizeInBits(); 806 807 bool IsPair = isHvxPairTy(VecTy); 808 MVT SingleTy = MVT::getVectorVT(ElemTy, (8*HwLen)/ElemWidth); 809 // The two single vectors that VecV consists of, if it's a pair. 810 SDValue V0, V1; 811 SDValue SingleV = VecV; 812 SDValue PickHi; 813 814 if (IsPair) { 815 V0 = DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, SingleTy, VecV); 816 V1 = DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, SingleTy, VecV); 817 818 SDValue HalfV = DAG.getConstant(SingleTy.getVectorNumElements(), 819 dl, MVT::i32); 820 PickHi = DAG.getSetCC(dl, MVT::i1, IdxV, HalfV, ISD::SETUGT); 821 if (isHvxSingleTy(SubTy)) { 822 if (const auto *CN = dyn_cast<const ConstantSDNode>(IdxV.getNode())) { 823 unsigned Idx = CN->getZExtValue(); 824 assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2); 825 unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi; 826 return DAG.getTargetInsertSubreg(SubIdx, dl, VecTy, VecV, SubV); 827 } 828 // If IdxV is not a constant, generate the two variants: with the 829 // SubV as the high and as the low subregister, and select the right 830 // pair based on the IdxV. 831 SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SubV, V1}); 832 SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SubV}); 833 return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo); 834 } 835 // The subvector being inserted must be entirely contained in one of 836 // the vectors V0 or V1. Set SingleV to the correct one, and update 837 // IdxV to be the index relative to the beginning of that vector. 838 SDValue S = DAG.getNode(ISD::SUB, dl, MVT::i32, IdxV, HalfV); 839 IdxV = DAG.getNode(ISD::SELECT, dl, MVT::i32, PickHi, S, IdxV); 840 SingleV = DAG.getNode(ISD::SELECT, dl, SingleTy, PickHi, V1, V0); 841 } 842 843 // The only meaningful subvectors of a single HVX vector are those that 844 // fit in a scalar register. 845 assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64); 846 // Convert IdxV to be index in bytes. 847 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode()); 848 if (!IdxN || !IdxN->isNullValue()) { 849 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, 850 DAG.getConstant(ElemWidth/8, dl, MVT::i32)); 851 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, IdxV); 852 } 853 // When inserting a single word, the rotation back to the original position 854 // would be by HwLen-Idx, but if two words are inserted, it will need to be 855 // by (HwLen-4)-Idx. 856 unsigned RolBase = HwLen; 857 if (VecTy.getSizeInBits() == 32) { 858 SDValue V = DAG.getBitcast(MVT::i32, SubV); 859 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, V); 860 } else { 861 SDValue V = DAG.getBitcast(MVT::i64, SubV); 862 SDValue R0 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, V); 863 SDValue R1 = DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, V); 864 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R0); 865 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, 866 DAG.getConstant(4, dl, MVT::i32)); 867 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R1); 868 RolBase = HwLen-4; 869 } 870 // If the vector wasn't ror'ed, don't ror it back. 871 if (RolBase != 4 || !IdxN || !IdxN->isNullValue()) { 872 SDValue RolV = DAG.getNode(ISD::SUB, dl, MVT::i32, 873 DAG.getConstant(RolBase, dl, MVT::i32), IdxV); 874 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, RolV); 875 } 876 877 if (IsPair) { 878 SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SingleV, V1}); 879 SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SingleV}); 880 return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo); 881 } 882 return SingleV; 883 } 884 885 SDValue 886 HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV, 887 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const { 888 MVT VecTy = ty(VecV); 889 MVT SubTy = ty(SubV); 890 assert(Subtarget.isHVXVectorType(VecTy, true)); 891 // VecV is an HVX vector predicate. SubV may be either an HVX vector 892 // predicate as well, or it can be a scalar predicate. 893 894 unsigned VecLen = VecTy.getVectorNumElements(); 895 unsigned HwLen = Subtarget.getVectorLength(); 896 assert(HwLen % VecLen == 0 && "Unexpected vector type"); 897 898 unsigned Scale = VecLen / SubTy.getVectorNumElements(); 899 unsigned BitBytes = HwLen / VecLen; 900 unsigned BlockLen = HwLen / Scale; 901 902 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 903 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV); 904 SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG); 905 SDValue ByteIdx; 906 907 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode()); 908 if (!IdxN || !IdxN->isNullValue()) { 909 ByteIdx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, 910 DAG.getConstant(BitBytes, dl, MVT::i32)); 911 ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteIdx); 912 } 913 914 // ByteVec is the target vector VecV rotated in such a way that the 915 // subvector should be inserted at index 0. Generate a predicate mask 916 // and use vmux to do the insertion. 917 MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); 918 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy, 919 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG); 920 ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG); 921 // Rotate ByteVec back, and convert to a vector predicate. 922 if (!IdxN || !IdxN->isNullValue()) { 923 SDValue HwLenV = DAG.getConstant(HwLen, dl, MVT::i32); 924 SDValue ByteXdi = DAG.getNode(ISD::SUB, dl, MVT::i32, HwLenV, ByteIdx); 925 ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteXdi); 926 } 927 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec); 928 } 929 930 SDValue 931 HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl, 932 MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const { 933 // Sign- and any-extending of a vector predicate to a vector register is 934 // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and 935 // a vector of 1s (where the 1s are of type matching the vector type). 936 assert(Subtarget.isHVXVectorType(ResTy)); 937 if (!ZeroExt) 938 return DAG.getNode(HexagonISD::Q2V, dl, ResTy, VecV); 939 940 assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements()); 941 SDValue True = DAG.getNode(HexagonISD::VSPLAT, dl, ResTy, 942 DAG.getConstant(1, dl, MVT::i32)); 943 SDValue False = getZero(dl, ResTy, DAG); 944 return DAG.getSelect(dl, ResTy, VecV, True, False); 945 } 946 947 SDValue 948 HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG) 949 const { 950 const SDLoc &dl(Op); 951 MVT VecTy = ty(Op); 952 953 unsigned Size = Op.getNumOperands(); 954 SmallVector<SDValue,128> Ops; 955 for (unsigned i = 0; i != Size; ++i) 956 Ops.push_back(Op.getOperand(i)); 957 958 if (VecTy.getVectorElementType() == MVT::i1) 959 return buildHvxVectorPred(Ops, dl, VecTy, DAG); 960 961 if (VecTy.getSizeInBits() == 16*Subtarget.getVectorLength()) { 962 ArrayRef<SDValue> A(Ops); 963 MVT SingleTy = typeSplit(VecTy).first; 964 SDValue V0 = buildHvxVectorReg(A.take_front(Size/2), dl, SingleTy, DAG); 965 SDValue V1 = buildHvxVectorReg(A.drop_front(Size/2), dl, SingleTy, DAG); 966 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, V0, V1); 967 } 968 969 return buildHvxVectorReg(Ops, dl, VecTy, DAG); 970 } 971 972 SDValue 973 HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG) 974 const { 975 // Vector concatenation of two integer (non-bool) vectors does not need 976 // special lowering. Custom-lower concats of bool vectors and expand 977 // concats of more than 2 vectors. 978 MVT VecTy = ty(Op); 979 const SDLoc &dl(Op); 980 unsigned NumOp = Op.getNumOperands(); 981 if (VecTy.getVectorElementType() != MVT::i1) { 982 if (NumOp == 2) 983 return Op; 984 // Expand the other cases into a build-vector. 985 SmallVector<SDValue,8> Elems; 986 for (SDValue V : Op.getNode()->ops()) 987 DAG.ExtractVectorElements(V, Elems); 988 // A vector of i16 will be broken up into a build_vector of i16's. 989 // This is a problem, since at the time of operation legalization, 990 // all operations are expected to be type-legalized, and i16 is not 991 // a legal type. If any of the extracted elements is not of a valid 992 // type, sign-extend it to a valid one. 993 for (unsigned i = 0, e = Elems.size(); i != e; ++i) { 994 SDValue V = Elems[i]; 995 MVT Ty = ty(V); 996 if (!isTypeLegal(Ty)) { 997 EVT NTy = getTypeToTransformTo(*DAG.getContext(), Ty); 998 if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 999 Elems[i] = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy, 1000 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy, 1001 V.getOperand(0), V.getOperand(1)), 1002 DAG.getValueType(Ty)); 1003 continue; 1004 } 1005 // A few less complicated cases. 1006 if (V.getOpcode() == ISD::Constant) 1007 Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy); 1008 else if (V.isUndef()) 1009 Elems[i] = DAG.getUNDEF(NTy); 1010 else 1011 llvm_unreachable("Unexpected vector element"); 1012 } 1013 } 1014 return DAG.getBuildVector(VecTy, dl, Elems); 1015 } 1016 1017 assert(VecTy.getVectorElementType() == MVT::i1); 1018 unsigned HwLen = Subtarget.getVectorLength(); 1019 assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0); 1020 1021 SDValue Op0 = Op.getOperand(0); 1022 1023 // If the operands are HVX types (i.e. not scalar predicates), then 1024 // defer the concatenation, and create QCAT instead. 1025 if (Subtarget.isHVXVectorType(ty(Op0), true)) { 1026 if (NumOp == 2) 1027 return DAG.getNode(HexagonISD::QCAT, dl, VecTy, Op0, Op.getOperand(1)); 1028 1029 ArrayRef<SDUse> U(Op.getNode()->ops()); 1030 SmallVector<SDValue,4> SV(U.begin(), U.end()); 1031 ArrayRef<SDValue> Ops(SV); 1032 1033 MVT HalfTy = typeSplit(VecTy).first; 1034 SDValue V0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy, 1035 Ops.take_front(NumOp/2)); 1036 SDValue V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy, 1037 Ops.take_back(NumOp/2)); 1038 return DAG.getNode(HexagonISD::QCAT, dl, VecTy, V0, V1); 1039 } 1040 1041 // Count how many bytes (in a vector register) each bit in VecTy 1042 // corresponds to. 1043 unsigned BitBytes = HwLen / VecTy.getVectorNumElements(); 1044 1045 SmallVector<SDValue,8> Prefixes; 1046 for (SDValue V : Op.getNode()->op_values()) { 1047 SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG); 1048 Prefixes.push_back(P); 1049 } 1050 1051 unsigned InpLen = ty(Op.getOperand(0)).getVectorNumElements(); 1052 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 1053 SDValue S = DAG.getConstant(InpLen*BitBytes, dl, MVT::i32); 1054 SDValue Res = getZero(dl, ByteTy, DAG); 1055 for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) { 1056 Res = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Res, S); 1057 Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]); 1058 } 1059 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, Res); 1060 } 1061 1062 SDValue 1063 HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG) 1064 const { 1065 // Change the type of the extracted element to i32. 1066 SDValue VecV = Op.getOperand(0); 1067 MVT ElemTy = ty(VecV).getVectorElementType(); 1068 const SDLoc &dl(Op); 1069 SDValue IdxV = Op.getOperand(1); 1070 if (ElemTy == MVT::i1) 1071 return extractHvxElementPred(VecV, IdxV, dl, ty(Op), DAG); 1072 1073 return extractHvxElementReg(VecV, IdxV, dl, ty(Op), DAG); 1074 } 1075 1076 SDValue 1077 HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG) 1078 const { 1079 const SDLoc &dl(Op); 1080 SDValue VecV = Op.getOperand(0); 1081 SDValue ValV = Op.getOperand(1); 1082 SDValue IdxV = Op.getOperand(2); 1083 MVT ElemTy = ty(VecV).getVectorElementType(); 1084 if (ElemTy == MVT::i1) 1085 return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG); 1086 1087 return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG); 1088 } 1089 1090 SDValue 1091 HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG) 1092 const { 1093 SDValue SrcV = Op.getOperand(0); 1094 MVT SrcTy = ty(SrcV); 1095 MVT DstTy = ty(Op); 1096 SDValue IdxV = Op.getOperand(1); 1097 unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue(); 1098 assert(Idx % DstTy.getVectorNumElements() == 0); 1099 (void)Idx; 1100 const SDLoc &dl(Op); 1101 1102 MVT ElemTy = SrcTy.getVectorElementType(); 1103 if (ElemTy == MVT::i1) 1104 return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG); 1105 1106 return extractHvxSubvectorReg(SrcV, IdxV, dl, DstTy, DAG); 1107 } 1108 1109 SDValue 1110 HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG) 1111 const { 1112 // Idx does not need to be a constant. 1113 SDValue VecV = Op.getOperand(0); 1114 SDValue ValV = Op.getOperand(1); 1115 SDValue IdxV = Op.getOperand(2); 1116 1117 const SDLoc &dl(Op); 1118 MVT VecTy = ty(VecV); 1119 MVT ElemTy = VecTy.getVectorElementType(); 1120 if (ElemTy == MVT::i1) 1121 return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG); 1122 1123 return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG); 1124 } 1125 1126 SDValue 1127 HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const { 1128 // Lower any-extends of boolean vectors to sign-extends, since they 1129 // translate directly to Q2V. Zero-extending could also be done equally 1130 // fast, but Q2V is used/recognized in more places. 1131 // For all other vectors, use zero-extend. 1132 MVT ResTy = ty(Op); 1133 SDValue InpV = Op.getOperand(0); 1134 MVT ElemTy = ty(InpV).getVectorElementType(); 1135 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) 1136 return LowerHvxSignExt(Op, DAG); 1137 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Op), ResTy, InpV); 1138 } 1139 1140 SDValue 1141 HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const { 1142 MVT ResTy = ty(Op); 1143 SDValue InpV = Op.getOperand(0); 1144 MVT ElemTy = ty(InpV).getVectorElementType(); 1145 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) 1146 return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG); 1147 return Op; 1148 } 1149 1150 SDValue 1151 HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const { 1152 MVT ResTy = ty(Op); 1153 SDValue InpV = Op.getOperand(0); 1154 MVT ElemTy = ty(InpV).getVectorElementType(); 1155 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) 1156 return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG); 1157 return Op; 1158 } 1159 1160 SDValue 1161 HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const { 1162 // Lower vector CTTZ into a computation using CTLZ (Hacker's Delight): 1163 // cttz(x) = bitwidth(x) - ctlz(~x & (x-1)) 1164 const SDLoc &dl(Op); 1165 MVT ResTy = ty(Op); 1166 SDValue InpV = Op.getOperand(0); 1167 assert(ResTy == ty(InpV)); 1168 1169 // Calculate the vectors of 1 and bitwidth(x). 1170 MVT ElemTy = ty(InpV).getVectorElementType(); 1171 unsigned ElemWidth = ElemTy.getSizeInBits(); 1172 // Using uint64_t because a shift by 32 can happen. 1173 uint64_t Splat1 = 0, SplatW = 0; 1174 assert(isPowerOf2_32(ElemWidth) && ElemWidth <= 32); 1175 for (unsigned i = 0; i != 32/ElemWidth; ++i) { 1176 Splat1 = (Splat1 << ElemWidth) | 1; 1177 SplatW = (SplatW << ElemWidth) | ElemWidth; 1178 } 1179 SDValue Vec1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy, 1180 DAG.getConstant(uint32_t(Splat1), dl, MVT::i32)); 1181 SDValue VecW = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy, 1182 DAG.getConstant(uint32_t(SplatW), dl, MVT::i32)); 1183 SDValue VecN1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy, 1184 DAG.getConstant(-1, dl, MVT::i32)); 1185 // Do not use DAG.getNOT, because that would create BUILD_VECTOR with 1186 // a BITCAST. Here we can skip the BITCAST (so we don't have to handle 1187 // it separately in custom combine or selection). 1188 SDValue A = DAG.getNode(ISD::AND, dl, ResTy, 1189 {DAG.getNode(ISD::XOR, dl, ResTy, {InpV, VecN1}), 1190 DAG.getNode(ISD::SUB, dl, ResTy, {InpV, Vec1})}); 1191 return DAG.getNode(ISD::SUB, dl, ResTy, 1192 {VecW, DAG.getNode(ISD::CTLZ, dl, ResTy, A)}); 1193 } 1194 1195 SDValue 1196 HexagonTargetLowering::LowerHvxMul(SDValue Op, SelectionDAG &DAG) const { 1197 MVT ResTy = ty(Op); 1198 assert(ResTy.isVector() && isHvxSingleTy(ResTy)); 1199 const SDLoc &dl(Op); 1200 SmallVector<int,256> ShuffMask; 1201 1202 MVT ElemTy = ResTy.getVectorElementType(); 1203 unsigned VecLen = ResTy.getVectorNumElements(); 1204 SDValue Vs = Op.getOperand(0); 1205 SDValue Vt = Op.getOperand(1); 1206 1207 switch (ElemTy.SimpleTy) { 1208 case MVT::i8: { 1209 // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...), 1210 // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo, 1211 // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...). 1212 MVT ExtTy = typeExtElem(ResTy, 2); 1213 unsigned MpyOpc = ElemTy == MVT::i8 ? Hexagon::V6_vmpybv 1214 : Hexagon::V6_vmpyhv; 1215 SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG); 1216 1217 // Discard high halves of the resulting values, collect the low halves. 1218 for (unsigned I = 0; I < VecLen; I += 2) { 1219 ShuffMask.push_back(I); // Pick even element. 1220 ShuffMask.push_back(I+VecLen); // Pick odd element. 1221 } 1222 VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG); 1223 SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG); 1224 return DAG.getBitcast(ResTy, BS); 1225 } 1226 case MVT::i16: 1227 // For i16 there is V6_vmpyih, which acts exactly like the MUL opcode. 1228 // (There is also V6_vmpyhv, which behaves in an analogous way to 1229 // V6_vmpybv.) 1230 return getInstr(Hexagon::V6_vmpyih, dl, ResTy, {Vs, Vt}, DAG); 1231 case MVT::i32: { 1232 // Use the following sequence for signed word multiply: 1233 // T0 = V6_vmpyiowh Vs, Vt 1234 // T1 = V6_vaslw T0, 16 1235 // T2 = V6_vmpyiewuh_acc T1, Vs, Vt 1236 SDValue S16 = DAG.getConstant(16, dl, MVT::i32); 1237 SDValue T0 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {Vs, Vt}, DAG); 1238 SDValue T1 = getInstr(Hexagon::V6_vaslw, dl, ResTy, {T0, S16}, DAG); 1239 SDValue T2 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy, 1240 {T1, Vs, Vt}, DAG); 1241 return T2; 1242 } 1243 default: 1244 break; 1245 } 1246 return SDValue(); 1247 } 1248 1249 SDValue 1250 HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const { 1251 MVT ResTy = ty(Op); 1252 assert(ResTy.isVector()); 1253 const SDLoc &dl(Op); 1254 SmallVector<int,256> ShuffMask; 1255 1256 MVT ElemTy = ResTy.getVectorElementType(); 1257 unsigned VecLen = ResTy.getVectorNumElements(); 1258 SDValue Vs = Op.getOperand(0); 1259 SDValue Vt = Op.getOperand(1); 1260 bool IsSigned = Op.getOpcode() == ISD::MULHS; 1261 1262 if (ElemTy == MVT::i8 || ElemTy == MVT::i16) { 1263 // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...), 1264 // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo, 1265 // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...). 1266 // For i16, use V6_vmpyhv, which behaves in an analogous way to 1267 // V6_vmpybv: results Lo and Hi are products of even/odd elements 1268 // respectively. 1269 MVT ExtTy = typeExtElem(ResTy, 2); 1270 unsigned MpyOpc = ElemTy == MVT::i8 1271 ? (IsSigned ? Hexagon::V6_vmpybv : Hexagon::V6_vmpyubv) 1272 : (IsSigned ? Hexagon::V6_vmpyhv : Hexagon::V6_vmpyuhv); 1273 SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG); 1274 1275 // Discard low halves of the resulting values, collect the high halves. 1276 for (unsigned I = 0; I < VecLen; I += 2) { 1277 ShuffMask.push_back(I+1); // Pick even element. 1278 ShuffMask.push_back(I+VecLen+1); // Pick odd element. 1279 } 1280 VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG); 1281 SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG); 1282 return DAG.getBitcast(ResTy, BS); 1283 } 1284 1285 assert(ElemTy == MVT::i32); 1286 SDValue S16 = DAG.getConstant(16, dl, MVT::i32); 1287 1288 if (IsSigned) { 1289 // mulhs(Vs,Vt) = 1290 // = [(Hi(Vs)*2^16 + Lo(Vs)) *s (Hi(Vt)*2^16 + Lo(Vt))] >> 32 1291 // = [Hi(Vs)*2^16 *s Hi(Vt)*2^16 + Hi(Vs) *su Lo(Vt)*2^16 1292 // + Lo(Vs) *us (Hi(Vt)*2^16 + Lo(Vt))] >> 32 1293 // = [Hi(Vs) *s Hi(Vt)*2^32 + Hi(Vs) *su Lo(Vt)*2^16 1294 // + Lo(Vs) *us Vt] >> 32 1295 // The low half of Lo(Vs)*Lo(Vt) will be discarded (it's not added to 1296 // anything, so it cannot produce any carry over to higher bits), 1297 // so everything in [] can be shifted by 16 without loss of precision. 1298 // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + Lo(Vs)*Vt >> 16] >> 16 1299 // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + V6_vmpyewuh(Vs,Vt)] >> 16 1300 // Denote Hi(Vs) = Vs': 1301 // = [Vs'*s Hi(Vt)*2^16 + Vs' *su Lo(Vt) + V6_vmpyewuh(Vt,Vs)] >> 16 1302 // = Vs'*s Hi(Vt) + (V6_vmpyiewuh(Vs',Vt) + V6_vmpyewuh(Vt,Vs)) >> 16 1303 SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, ResTy, {Vt, Vs}, DAG); 1304 // Get Vs': 1305 SDValue S0 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {Vs, S16}, DAG); 1306 SDValue T1 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy, 1307 {T0, S0, Vt}, DAG); 1308 // Shift by 16: 1309 SDValue S2 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {T1, S16}, DAG); 1310 // Get Vs'*Hi(Vt): 1311 SDValue T2 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {S0, Vt}, DAG); 1312 // Add: 1313 SDValue T3 = DAG.getNode(ISD::ADD, dl, ResTy, {S2, T2}); 1314 return T3; 1315 } 1316 1317 // Unsigned mulhw. (Would expansion using signed mulhw be better?) 1318 1319 auto LoVec = [&DAG,ResTy,dl] (SDValue Pair) { 1320 return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, ResTy, Pair); 1321 }; 1322 auto HiVec = [&DAG,ResTy,dl] (SDValue Pair) { 1323 return DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, ResTy, Pair); 1324 }; 1325 1326 MVT PairTy = typeJoin({ResTy, ResTy}); 1327 SDValue P = getInstr(Hexagon::V6_lvsplatw, dl, ResTy, 1328 {DAG.getConstant(0x02020202, dl, MVT::i32)}, DAG); 1329 // Multiply-unsigned halfwords: 1330 // LoVec = Vs.uh[2i] * Vt.uh[2i], 1331 // HiVec = Vs.uh[2i+1] * Vt.uh[2i+1] 1332 SDValue T0 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, Vt}, DAG); 1333 // The low halves in the LoVec of the pair can be discarded. They are 1334 // not added to anything (in the full-precision product), so they cannot 1335 // produce a carry into the higher bits. 1336 SDValue T1 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {LoVec(T0), S16}, DAG); 1337 // Swap low and high halves in Vt, and do the halfword multiplication 1338 // to get products Vs.uh[2i] * Vt.uh[2i+1] and Vs.uh[2i+1] * Vt.uh[2i]. 1339 SDValue D0 = getInstr(Hexagon::V6_vdelta, dl, ResTy, {Vt, P}, DAG); 1340 SDValue T2 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, D0}, DAG); 1341 // T2 has mixed products of halfwords: Lo(Vt)*Hi(Vs) and Hi(Vt)*Lo(Vs). 1342 // These products are words, but cannot be added directly because the 1343 // sums could overflow. Add these products, by halfwords, where each sum 1344 // of a pair of halfwords gives a word. 1345 SDValue T3 = getInstr(Hexagon::V6_vadduhw, dl, PairTy, 1346 {LoVec(T2), HiVec(T2)}, DAG); 1347 // Add the high halfwords from the products of the low halfwords. 1348 SDValue T4 = DAG.getNode(ISD::ADD, dl, ResTy, {T1, LoVec(T3)}); 1349 SDValue T5 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {T4, S16}, DAG); 1350 SDValue T6 = DAG.getNode(ISD::ADD, dl, ResTy, {HiVec(T0), HiVec(T3)}); 1351 SDValue T7 = DAG.getNode(ISD::ADD, dl, ResTy, {T5, T6}); 1352 return T7; 1353 } 1354 1355 SDValue 1356 HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const { 1357 // Sign- and zero-extends are legal. 1358 assert(Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG); 1359 return DAG.getZeroExtendVectorInReg(Op.getOperand(0), SDLoc(Op), ty(Op)); 1360 } 1361 1362 SDValue 1363 HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const { 1364 if (SDValue S = getVectorShiftByInt(Op, DAG)) 1365 return S; 1366 return Op; 1367 } 1368 1369 SDValue 1370 HexagonTargetLowering::SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const { 1371 assert(!Op.isMachineOpcode()); 1372 SmallVector<SDValue,2> OpsL, OpsH; 1373 const SDLoc &dl(Op); 1374 1375 auto SplitVTNode = [&DAG,this] (const VTSDNode *N) { 1376 MVT Ty = typeSplit(N->getVT().getSimpleVT()).first; 1377 SDValue TV = DAG.getValueType(Ty); 1378 return std::make_pair(TV, TV); 1379 }; 1380 1381 for (SDValue A : Op.getNode()->ops()) { 1382 VectorPair P = Subtarget.isHVXVectorType(ty(A), true) 1383 ? opSplit(A, dl, DAG) 1384 : std::make_pair(A, A); 1385 // Special case for type operand. 1386 if (Op.getOpcode() == ISD::SIGN_EXTEND_INREG) { 1387 if (const auto *N = dyn_cast<const VTSDNode>(A.getNode())) 1388 P = SplitVTNode(N); 1389 } 1390 OpsL.push_back(P.first); 1391 OpsH.push_back(P.second); 1392 } 1393 1394 MVT ResTy = ty(Op); 1395 MVT HalfTy = typeSplit(ResTy).first; 1396 SDValue L = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsL); 1397 SDValue H = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsH); 1398 SDValue S = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, L, H); 1399 return S; 1400 } 1401 1402 SDValue 1403 HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const { 1404 LSBaseSDNode *BN = cast<LSBaseSDNode>(Op.getNode()); 1405 assert(BN->isUnindexed()); 1406 MVT MemTy = BN->getMemoryVT().getSimpleVT(); 1407 if (!isHvxPairTy(MemTy)) 1408 return Op; 1409 1410 const SDLoc &dl(Op); 1411 unsigned HwLen = Subtarget.getVectorLength(); 1412 MVT SingleTy = typeSplit(MemTy).first; 1413 SDValue Chain = BN->getChain(); 1414 SDValue Base0 = BN->getBasePtr(); 1415 SDValue Base1 = DAG.getMemBasePlusOffset(Base0, HwLen, dl); 1416 1417 MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr; 1418 if (MachineMemOperand *MMO = BN->getMemOperand()) { 1419 MachineFunction &MF = DAG.getMachineFunction(); 1420 MOp0 = MF.getMachineMemOperand(MMO, 0, HwLen); 1421 MOp1 = MF.getMachineMemOperand(MMO, HwLen, HwLen); 1422 } 1423 1424 unsigned MemOpc = BN->getOpcode(); 1425 SDValue NewOp; 1426 1427 if (MemOpc == ISD::LOAD) { 1428 SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0); 1429 SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1); 1430 NewOp = DAG.getMergeValues( 1431 { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, Load0, Load1), 1432 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 1433 Load0.getValue(1), Load1.getValue(1)) }, dl); 1434 } else { 1435 assert(MemOpc == ISD::STORE); 1436 VectorPair Vals = opSplit(cast<StoreSDNode>(Op)->getValue(), dl, DAG); 1437 SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0); 1438 SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1); 1439 NewOp = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store0, Store1); 1440 } 1441 1442 return NewOp; 1443 } 1444 1445 SDValue 1446 HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const { 1447 unsigned Opc = Op.getOpcode(); 1448 bool IsPairOp = isHvxPairTy(ty(Op)) || 1449 llvm::any_of(Op.getNode()->ops(), [this] (SDValue V) { 1450 return isHvxPairTy(ty(V)); 1451 }); 1452 1453 if (IsPairOp) { 1454 switch (Opc) { 1455 default: 1456 break; 1457 case ISD::LOAD: 1458 case ISD::STORE: 1459 return SplitHvxMemOp(Op, DAG); 1460 case ISD::CTPOP: 1461 case ISD::CTLZ: 1462 case ISD::CTTZ: 1463 case ISD::MUL: 1464 case ISD::MULHS: 1465 case ISD::MULHU: 1466 case ISD::AND: 1467 case ISD::OR: 1468 case ISD::XOR: 1469 case ISD::SRA: 1470 case ISD::SHL: 1471 case ISD::SRL: 1472 case ISD::SETCC: 1473 case ISD::VSELECT: 1474 case ISD::SIGN_EXTEND_INREG: 1475 return SplitHvxPairOp(Op, DAG); 1476 } 1477 } 1478 1479 switch (Opc) { 1480 default: 1481 break; 1482 case ISD::BUILD_VECTOR: return LowerHvxBuildVector(Op, DAG); 1483 case ISD::CONCAT_VECTORS: return LowerHvxConcatVectors(Op, DAG); 1484 case ISD::INSERT_SUBVECTOR: return LowerHvxInsertSubvector(Op, DAG); 1485 case ISD::INSERT_VECTOR_ELT: return LowerHvxInsertElement(Op, DAG); 1486 case ISD::EXTRACT_SUBVECTOR: return LowerHvxExtractSubvector(Op, DAG); 1487 case ISD::EXTRACT_VECTOR_ELT: return LowerHvxExtractElement(Op, DAG); 1488 1489 case ISD::ANY_EXTEND: return LowerHvxAnyExt(Op, DAG); 1490 case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG); 1491 case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG); 1492 case ISD::CTTZ: return LowerHvxCttz(Op, DAG); 1493 case ISD::SRA: 1494 case ISD::SHL: 1495 case ISD::SRL: return LowerHvxShift(Op, DAG); 1496 case ISD::MUL: return LowerHvxMul(Op, DAG); 1497 case ISD::MULHS: 1498 case ISD::MULHU: return LowerHvxMulh(Op, DAG); 1499 case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG); 1500 case ISD::SETCC: 1501 case ISD::INTRINSIC_VOID: return Op; 1502 // Unaligned loads will be handled by the default lowering. 1503 case ISD::LOAD: return SDValue(); 1504 } 1505 #ifndef NDEBUG 1506 Op.dumpr(&DAG); 1507 #endif 1508 llvm_unreachable("Unhandled HVX operation"); 1509 } 1510 1511 bool 1512 HexagonTargetLowering::isHvxOperation(SDValue Op) const { 1513 // If the type of the result, or any operand type are HVX vector types, 1514 // this is an HVX operation. 1515 return Subtarget.isHVXVectorType(ty(Op), true) || 1516 llvm::any_of(Op.getNode()->ops(), 1517 [this] (SDValue V) { 1518 return Subtarget.isHVXVectorType(ty(V), true); 1519 }); 1520 } 1521