Lines Matching defs:LU
829 /// folded in \p LU at isel time.
831 /// This function returns true if \p LU can accommodate what \p F
838 const LSRUse &LU, const Formula &F);
839 // Get the cost of the scaling factor used in F for LU.
841 const LSRUse &LU, const Formula &F);
888 const LSRUse &LU,
980 const LSRUse &LU,
1009 NumBaseParts - (1 + (F.Scale && isAMCompletelyFolded(TTI, LU, F)));
1013 ScaleCost += getScalingFactorCost(TTI, LU, F);
1468 const LSRUse &LU, const Formula &F) {
1469 return isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1470 LU.AccessTy, F.BaseGV, F.BaseOffset, F.HasBaseReg,
1475 const LSRUse &LU, const Formula &F) {
1481 if (!isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1482 LU.AccessTy, F))
1485 switch (LU.Kind) {
1489 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MinOffset, F.HasBaseReg,
1490 F.Scale, LU.AccessTy.AddrSpace);
1492 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MaxOffset, F.HasBaseReg,
1493 F.Scale, LU.AccessTy.AddrSpace);
1690 bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg,
1696 void DeleteUse(LSRUse &LU, size_t LUIdx);
1700 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1701 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1703 bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
1707 void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
1710 void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
1713 void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
1714 void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
1717 void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
1718 void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
1722 void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
1723 void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
1724 void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
1725 void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
1752 const LSRUse &LU,
2206 bool LSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset,
2209 int64_t NewMinOffset = LU.MinOffset;
2210 int64_t NewMaxOffset = LU.MaxOffset;
2216 if (LU.Kind != Kind)
2223 if (AccessTy != LU.AccessTy)
2228 if (NewOffset < LU.MinOffset) {
2230 LU.MaxOffset - NewOffset, HasBaseReg))
2233 } else if (NewOffset > LU.MaxOffset) {
2235 NewOffset - LU.MinOffset, HasBaseReg))
2241 LU.MinOffset = NewMinOffset;
2242 LU.MaxOffset = NewMaxOffset;
2243 LU.AccessTy = NewAccessTy;
2244 if (NewOffset != LU.Offsets.back())
2245 LU.Offsets.push_back(NewOffset);
2270 LSRUse &LU = Uses[LUIdx];
2271 if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy))
2280 LSRUse &LU = Uses[LUIdx];
2284 if (LU.Offsets.empty() || Offset != LU.Offsets.back())
2285 LU.Offsets.push_back(Offset);
2287 LU.MinOffset = Offset;
2288 LU.MaxOffset = Offset;
2293 void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) {
2294 if (&LU != &Uses.back())
2295 std::swap(LU, Uses.back());
2309 LSRUse &LU = Uses[LUIdx];
2315 if (&LU != &OrigLU &&
2316 LU.Kind != LSRUse::ICmpZero &&
2317 LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy &&
2318 LU.WidestFixupType == OrigLU.WidestFixupType &&
2319 LU.HasFormulaWithSameRegs(OrigF)) {
2321 for (const Formula &F : LU.Formulae) {
2330 return &LU;
2997 LSRUse &LU = Uses[LF.LUIdx];
2998 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
2999 if (!LU.WidestFixupType ||
3000 SE.getTypeSizeInBits(LU.WidestFixupType) <
3002 LU.WidestFixupType = LF.OperandValToReplace->getType();
3005 if (LU.Formulae.empty()) {
3006 InsertInitialFormula(S, LU, LF.LUIdx);
3007 CountRegisters(LU.Formulae.back(), LF.LUIdx);
3017 LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) {
3020 LU.RigidFormula = true;
3024 bool Inserted = InsertFormula(LU, LUIdx, F);
3032 LSRUse &LU, size_t LUIdx) {
3036 bool Inserted = InsertFormula(LU, LUIdx, F);
3050 bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) {
3052 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) &&
3054 if (!LU.InsertFormula(F))
3140 LSRUse &LU = Uses[LF.LUIdx];
3141 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3142 if (!LU.WidestFixupType ||
3143 SE.getTypeSizeInBits(LU.WidestFixupType) <
3145 LU.WidestFixupType = LF.OperandValToReplace->getType();
3146 InsertSupplementalFormula(US, LU, LF.LUIdx);
3147 CountRegisters(LU.Formulae.back(), Uses.size() - 1);
3216 void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
3240 if (isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3241 LU.AccessTy, *J, Base.getNumRegs() > 1))
3253 isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3254 LU.AccessTy, InnerAddOps[0], Base.getNumRegs() > 1))
3291 if (InsertFormula(LU, LUIdx, F))
3294 GenerateReassociations(LU, LUIdx, LU.Formulae.back(), Depth + 1);
3299 void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx,
3307 GenerateReassociationsImpl(LU, LUIdx, Base, Depth, i);
3310 GenerateReassociationsImpl(LU, LUIdx, Base, Depth,
3316 void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
3343 (void)InsertFormula(LU, LUIdx, F);
3349 void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
3358 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
3364 (void)InsertFormula(LU, LUIdx, F);
3368 void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx,
3374 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, i);
3376 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, /* Idx */ -1,
3382 LSRUse &LU, unsigned LUIdx, const Formula &Base,
3388 if (isLegalUse(TTI, LU.MinOffset - Offset, LU.MaxOffset - Offset, LU.Kind,
3389 LU.AccessTy, F)) {
3405 (void)InsertFormula(LU, LUIdx, F);
3414 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
3420 (void)InsertFormula(LU, LUIdx, F);
3424 void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx,
3429 Worklist.push_back(LU.MinOffset);
3430 if (LU.MaxOffset != LU.MinOffset)
3431 Worklist.push_back(LU.MaxOffset);
3434 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, i);
3436 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, /* Idx */ -1,
3442 void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
3444 if (LU.Kind != LSRUse::ICmpZero) return;
3452 if (LU.MinOffset != LU.MaxOffset) return;
3470 int64_t Offset = LU.MinOffset;
3474 if (Offset / Factor != LU.MinOffset)
3485 if (!isLegalUse(TTI, Offset, Offset, LU.Kind, LU.AccessTy, F))
3489 F.BaseOffset = (uint64_t)F.BaseOffset + Offset - LU.MinOffset;
3521 (void)InsertFormula(LU, LUIdx, F);
3528 void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
3545 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
3549 if (LU.Kind == LSRUse::Basic &&
3550 isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LSRUse::Special,
3551 LU.AccessTy, Base) &&
3552 LU.AllFixupsOutsideLoop)
3553 LU.Kind = LSRUse::Special;
3559 if (LU.Kind == LSRUse::ICmpZero &&
3581 (void)InsertFormula(LU, LUIdx, F);
3588 void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
3610 (void)InsertFormula(LU, LUIdx, F);
3723 LSRUse &LU = Uses[LUIdx];
3732 for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) {
3733 Formula F = LU.Formulae[L];
3748 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
3764 (void)InsertFormula(LU, LUIdx, NewF);
3773 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset,
3774 LU.Kind, LU.AccessTy, NewF)) {
3796 (void)InsertFormula(LU, LUIdx, NewF);
3811 LSRUse &LU = Uses[LUIdx];
3812 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
3813 GenerateReassociations(LU, LUIdx, LU.Formulae[i]);
3814 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
3815 GenerateCombinations(LU, LUIdx, LU.Formulae[i]);
3818 LSRUse &LU = Uses[LUIdx];
3819 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
3820 GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]);
3821 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
3822 GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]);
3823 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
3824 GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]);
3825 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
3826 GenerateScales(LU, LUIdx, LU.Formulae[i]);
3829 LSRUse &LU = Uses[LUIdx];
3830 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
3831 GenerateTruncates(LU, LUIdx, LU.Formulae[i]);
3858 LSRUse &LU = Uses[LUIdx];
3859 DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs()); dbgs() << '\n');
3862 for (size_t FIdx = 0, NumForms = LU.Formulae.size();
3864 Formula &F = LU.Formulae[FIdx];
3875 CostF.RateFormula(TTI, F, Regs, VisitedRegs, L, LU.Offsets, SE, DT, LU,
3905 Formula &Best = LU.Formulae[P.first->second];
3909 CostBest.RateFormula(TTI, Best, Regs, VisitedRegs, L, LU.Offsets, SE,
3910 DT, LU);
3921 LU.DeleteFormula(F);
3929 LU.RecomputeRegs(LUIdx, RegUses);
3950 for (const LSRUse &LU : Uses) {
3951 size_t FSize = LU.Formulae.size();
3975 LSRUse &LU = Uses[LUIdx];
3977 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
3978 Formula &F = LU.Formulae[i];
3989 if (LU.HasFormulaWithSameRegs(NewF)) {
3991 LU.DeleteFormula(F);
4004 if (LU.HasFormulaWithSameRegs(NewF)) {
4007 LU.DeleteFormula(F);
4018 LU.RecomputeRegs(LUIdx, RegUses);
4039 LSRUse &LU = Uses[LUIdx];
4040 for (const Formula &F : LU.Formulae) {
4044 LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU);
4049 LU.Kind, LU.AccessTy))
4052 DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << '\n');
4054 LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop;
4094 DeleteUse(LU, LUIdx);
4158 LSRUse &LU = Uses[LUIdx];
4159 if (!LU.Regs.count(Best)) continue;
4162 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4163 Formula &F = LU.Formulae[i];
4166 LU.DeleteFormula(F);
4176 LU.RecomputeRegs(LUIdx, RegUses);
4212 const LSRUse &LU = Uses[Workspace.size()];
4220 if (LU.Regs.count(S))
4225 for (const Formula &F : LU.Formulae) {
4249 NewCost.RateFormula(TTI, F, NewRegs, VisitedRegs, L, LU.Offsets, SE, DT,
4250 LU);
4371 const LSRUse &LU,
4379 if (LU.Kind == LSRUse::ICmpZero)
4438 const LSRUse &LU = Uses[LF.LUIdx];
4439 if (LU.RigidFormula)
4444 IP = AdjustInsertPositionForExpand(IP, LF, LU, Rewriter);
4490 if (LU.Kind == LSRUse::ICmpZero) {
4509 if (!Ops.empty() && LU.Kind == LSRUse::Address &&
4510 isAMCompletelyFolded(TTI, LU, F)) {
4545 if (LU.Kind == LSRUse::ICmpZero) {
4581 if (LU.Kind == LSRUse::ICmpZero) {
4867 for (const LSRUse &LU : Uses) {
4868 for (const Formula &F : LU.Formulae)
4869 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
4909 for (const LSRUse &LU : Uses) {
4911 LU.print(OS);
4913 for (const Formula &F : LU.Formulae) {
