Lines Matching defs:Load
48 STATISTIC(OpsNarrowed , "Number of load/op/store narrowed");
49 STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int");
50 STATISTIC(SlicedLoads, "Number of load sliced");
72 /// Hidden option to stress test load slicing, i.e., when this option
73 /// is enabled, load slicing bypasses most of its profitability guards.
75 StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
76 cl::desc("Bypass the profitability model of load "
81 MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
117 // AA - Used for DAG load/store alias analysis.
192 /// \brief Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
193 /// load.
197 /// \param EltNo index of the vector element to load.
198 /// \param OriginalLoad load that EVE came from to be replaced.
202 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
504 // arithmetic from an indexed load).
888 void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
889 SDLoc dl(Load);
890 EVT VT = Load->getValueType(0);
894 Load->dump(&DAG);
899 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
900 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
901 deleteAndRecombine(Load);
1295 // If the root changed (e.g. it was a dead load, update the root).
1372 case ISD::LOAD: return visitLOAD(N);
1430 case ISD::LOAD:
2874 // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
2875 // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
2876 // already be zero by virtue of the width of the base type of the load.
2881 N0.getOperand(0).getOpcode() == ISD::LOAD) ||
2882 N0.getOpcode() == ISD::LOAD) {
2883 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
2933 Load->getValueType(0),
2934 Load->getMemoryVT());
2940 Constant.zextOrTrunc(Load->getMemoryVT().getScalarType().getSizeInBits());
2943 switch (Load->getExtensionType()) {
2951 // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
2953 SDValue NewLoad(Load, 0);
2954 if (Load->getExtensionType() == ISD::EXTLOAD) {
2955 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
2956 Load->getValueType(0), SDLoc(Load),
2957 Load->getChain(), Load->getBasePtr(),
2958 Load->getOffset(), Load->getMemoryVT(),
2959 Load->getMemOperand());
2961 if (Load->getNumValues() == 3) {
2965 CombineTo(Load, To, 3, true);
2967 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
2971 // Fold the AND away, taking care not to fold to the old load node if we
2973 CombineTo(N, (N0.getNode() == Load
2979 // fold (and (load x), 255) -> (zextload x, i8)
2982 if (N1C && (N0.getOpcode() == ISD::LOAD ||
2984 N0.getOperand(0).getOpcode() == ISD::LOAD))) {
3010 // Do not change the width of a volatile load.
3022 // to load the correct bytes. For little endian systems, we merely
3035 SDValue Load =
3042 CombineTo(LN0, Load, Load.getValue(1));
3083 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
4624 // Attempt to convert a srl of a load into a narrower zero-extending load.
5139 // Build a factor node to remember that this load is independent of the
5350 // ExtendUsesToFormExtLoad - Trying to extend uses of a load to enable this:
5351 // "fold ({s|z|a}ext (load x)) -> ({s|z|a}ext (truncate ({s|z|a}extload x)))"
5445 // fold (sext (load x)) to multiple smaller sextloads; same for zext.
5447 // (v8i32 (sext (v8i16 (load x))))
5451 // Where uses of the original load, i.e.:
5452 // (v8i16 (load x))
5462 if (N0->getOpcode() != ISD::LOAD)
5521 // Replace uses of the original load (before extension)
5546 // fold (sext (truncate (load x))) -> (sext (smaller load x))
5547 // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n)))
5596 // fold (sext (load x)) -> (sext (truncate (sextload x)))
5625 // fold (sext (load x)) to multiple smaller sextloads.
5651 // fold (sext (and/or/xor (load x), cst)) ->
5828 // fold (zext (truncate (load x))) -> (zext (smaller load x))
5829 // fold (zext (truncate (srl (load x), c))) -> (zext (small load (x+c/n)))
5847 // fold (zext (truncate (load x))) -> (zext (smaller load x))
5848 // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n)))
5892 // fold (zext (load x)) -> (zext (truncate (zextload x)))
5922 // fold (zext (load x)) to multiple smaller zextloads.
5927 // fold (zext (and/or/xor (load x), cst)) ->
6082 // fold (aext (truncate (load x))) -> (aext (smaller load x))
6083 // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n)))
6126 // fold (aext (load x)) -> (aext (truncate (extload x)))
6127 // None of the supported targets knows how to perform load and any_ext
6156 if (N0.getOpcode() == ISD::LOAD &&
6260 /// If the result of a wider load is shifted to right of N bits and then
6262 /// the narrower type, transform it to a narrower load from address + N / num of
6264 /// to form a extending load.
6308 // Is the load width a multiple of size of VT?
6313 // At this point, we must have a load or else we can't do the transform.
6323 // accessing any of the loaded bytes. If the load was a zextload/extload
6330 // If the load is shifted left (and the result isn't shifted back right),
6341 // If we haven't found a load, we can't narrow it. Don't transform one with
6342 // multiple uses, this would require adding a new load.
6346 // Don't change the width of a volatile load.
6351 // Verify that we are actually reducing a load width here.
6355 // For the transform to be legal, the load must produce only two values
6357 // load, for example, which produces an extra value. Otherwise the
6363 // If the load that we're shrinking is an extload and we're not just
6364 // discarding the extension we can't simply shrink the load. Bail.
6380 // load the correct bytes.
6394 SDValue Load;
6396 Load = DAG.getLoad(VT, SDLoc(N0), LN0->getChain(), NewPtr,
6401 Load = DAG.getExtLoad(ExtType, SDLoc(N0), VT, LN0->getChain(),NewPtr,
6406 // Replace the old load's chain with the new load's chain.
6408 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
6411 SDValue Result = Load;
6472 // fold (sext_in_reg (load x)) -> (smaller sextload x)
6473 // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits))
6509 // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use
6684 // fold (truncate (load x)) -> (smaller load x)
6685 // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
6690 // Handle the case where the load remains an extending load even
6762 /// build_pair (load, load) -> load
6763 /// if load locations are consecutive.
6786 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)))
6833 // fold (conv (load x)) -> (load (conv*)x)
6834 // If the resultant load doesn't need a higher alignment than the original!
6836 // Do not change the width of a volatile load.
6841 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) &&
6849 SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(),
6854 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
6855 return Load;
6924 // bitconvert(build_pair(ld, ld)) -> ld iff load locations are consecutive.
8183 // fold (fpext (load x)) -> (fpext (fptrunc (extload x)))
8552 /// Return true if 'Use' is a load or a store that uses N as its base pointer
8553 /// and that N may be folded in the load / store addressing mode.
8592 /// Try turning a load/store into a pre-indexed load/store when the base
8593 /// pointer is an add or subtract and it has other uses besides the load/store.
8594 /// After the transformation, the new indexed load/store has effectively folded
8596 /// new load/store.
8647 // Don't create a indexed load / store with zero offset.
8652 // Try turning it into a pre-indexed load / store except when:
8658 // 4) All uses are load / store ops that use it as old base ptr.
8773 // x1 * offset1 + y1 * ptr0 = t1 (the indexed load/store)
8776 // indexed load/store and the expresion that needs to be re-written.
8817 /// Try to combine a load/store with a add/sub of the base pointer node into a
8818 /// post-indexed load/store. The transformation folded the add/subtract into the
8819 /// new indexed load/store effectively and all of its uses are redirected to the
8820 /// new load/store.
8861 // Don't create a indexed load / store with zero offset.
8866 // Try turning it into a post-indexed load / store except when
8867 // 1) All uses are load / store ops that use it as base ptr (and
8882 // If all the uses are load / store addresses, then don't do the
8945 // Some backends use TargetConstants for load offsets, but don't expect
8967 // If load is not volatile and there are no uses of the loaded value (and
8969 // chain value into uses of the chain input (i.e. delete the dead load).
8975 // v1, chain2 = load chain1, loc
8976 // v2, chain3 = load chain2, loc
8978 // Now we replace use of chain2 with chain1. This makes the second load
8979 // isomorphic to the one we are deleting, and thus makes this load live.
8997 // If this load has an opaque TargetConstant offset, then we cannot split
9030 // If this load is directly stored, replace the load value with the stored
9043 // Try to infer better alignment information than the load already has.
9093 // Replace uses with load result and token factor. Don't add users
9099 // Try transforming N to an indexed load.
9112 /// \brief Helper structure used to slice a load in smaller loads.
9114 /// Origin = load Ty1, Base
9119 /// Slice = load SliceTy, Base + SliceOffset
9154 /// load with the gain from this slice.
9208 // The original load instruction.
9210 // The right shift amount in bits from the original load.
9228 assert(Origin && "No original load to compare against.");
9255 /// \brief Get the alignment of the load used for this slice.
9270 // Offsets are for indexed load only, we do not handle that.
9281 // Check that the load is legal for this type.
9282 if (!TLI.isOperationLegal(ISD::LOAD, SliceType))
9330 /// \brief Generate the sequence of instructions to load the slice
9335 /// \return The last instruction of the sequence used to load the slice.
9353 // Create the load for the slice.
9369 /// i = load i32
9383 if (ArgRC == ResRC || !TLI.isOperationLegal(ISD::LOAD, ResVT))
9394 // Check if it will be merged with the load.
9402 // 2. Check that the load is a legal operation for that type.
9403 if (!TLI.isOperationLegal(ISD::LOAD, ResVT))
9465 // to be placed in a paired load.
9510 /// whose cost (\see LoadedSlice::Cost) is smaller than the original load (3).
9533 // The original code has one big load.
9546 // If the target supports paired load, adjust the cost accordingly.
9551 /// \brief If the given load, \p LI, is used only by trunc or trunc(lshr)
9556 /// \pre LI is a simple load (i.e., not an atomic or volatile load).
9572 // Check if this load is used as several smaller chunks of bits.
9597 // Otherwise the load cannot be represented in LLVM IR.
9628 // Rewrite each chain to use an independent load.
9629 // By construction, each chain can be represented by a unique load.
9639 if (SliceInst.getNode()->getOpcode() != ISD::LOAD)
9641 assert(SliceInst->getOpcode() == ISD::LOAD &&
9652 /// Check to see if V is (and load (ptr), imm), where the load is having
9669 // The store should be chained directly to the load or be an operand of a
9784 /// Look for sequence of load / op / store where op is one of 'or', 'xor', and
9786 /// narrowing the load and store if it would end up being a win for performance
9803 // If this is "store (or X, Y), P" and X is "(and (load P), cst)", where cst
9806 // load + replace + store sequence with a single (narrower) store, which makes
9807 // the load dead.
9837 // Find the type to narrow it the load / op / store to.
9849 // The narrowing should be profitable, the load/store operation should be
9873 // load the correct bytes.
9911 /// For a given floating point load / store pair, if the load value isn't used
9913 /// load / store operations if the target deems the transformation profitable.
9932 if (!TLI.isOperationLegal(ISD::LOAD, IntVT) ||
9934 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) ||
9972 /// (load (i64 add (i64 copyfromreg %c)
9973 /// (i64 signextend (add (i8 load %index)
9977 /// (load (i64 add (i64 copyfromreg %c)
9978 /// (i64 signextend (i32 add (i32 signextend (i8 load %index))
10088 // can be materialized without a load.
10131 // Create the new Load and Store operations.
10260 // memory operand. If the next operand is a load save it and use alias
10274 // Save the load node for later. Continue the scan.
10318 // We found a load that alias with this store. Stop the sequence.
10392 // It should be possible to handle mixed sources, but load sources need
10410 // wide load and a single wide store.
10412 // Look for load nodes which are used by the stored values.
10461 // If we have load/store pair instructions and we only have two values,
10469 // load memory address. These variables hold the index in the store node
10536 // Find if it is better to use vectors or integers to load and store
10562 // Replace one of the loads with the new load.
10567 // Remove the rest of the load chains.
10569 // Replace all chain users of the old load nodes with the chain of the new
10570 // load node.
10701 // Try transforming a pair floating point load / store ops to integer
10702 // load / store ops.
10772 // If this is a load followed by a store to the same location, then the store
10777 // There can't be any side effects between the load and store, such as
10909 if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, VecEltVT))
10938 // replace an extractelement of a load with a load.
10941 // use of the load; that's okay because we don't want to perform this
10943 SDValue Load;
10946 // If the result type of vextract is wider than the load, then issue an
10947 // extending load instead.
10952 Load = DAG.getExtLoad(
10956 Chain = Load.getValue(1);
10958 Load = DAG.getLoad(
10962 Chain = Load.getValue(1);
10964 Load = DAG.getNode(ISD::TRUNCATE, SDLoc(EVE), ResultVT, Load);
10966 Load = DAG.getNode(ISD::BITCAST, SDLoc(EVE), ResultVT, Load);
10970 SDValue To[] = { Load, Chain };
10974 AddToWorklist(Load.getNode());
10975 AddUsersToWorklist(Load.getNode()); // Add users too
11054 // If the result of load has to be truncated, then it's not necessarily
11060 // Don't duplicate a load with other uses.
11073 // (vextract (vN[if]M load $addr), i) -> ([if]M load $addr + i * size)
11087 // (vextract (v4f32 load $addr), c) -> (f32 load $addr+c*size)
11088 // (vextract (v4f32 s2v (f32 load $addr)), c) -> (f32 load $addr+c*size)
11089 // (vextract (v4f32 shuffle (load $addr), <1,u,u,u>), 0) -> (f32 load $addr)
11101 // Don't duplicate a load with other uses.
11107 // (vextract (vector_shuffle (load $addr), v2, <1, u, u, u>), 1)
11109 // (load $addr+1*size)
11111 // Don't duplicate a load with other uses.
11126 // Don't duplicate a load with other uses.
11139 // Make sure we found a non-volatile load and the extractelement is
12557 // If this is a load and the token chain is identical, replace the select
12558 // of two loads with a load through a select of the address to load from.
12561 if (LHS.getOpcode() == ISD::LOAD) {
12587 // Check that the select condition doesn't reach either load. If so,
12622 SDValue Load;
12624 // but the new load must be the minimum (most restrictive) alignment of the
12629 Load = DAG.getLoad(TheSelect->getValueType(0),
12636 Load = DAG.getExtLoad(LLD->getExtensionType() == ISD::EXTLOAD ?
12646 // Users of the select now use the result of the load.
12647 CombineTo(TheSelect, Load);
12649 // Users of the old loads now use the new load's chain. We know the
12650 load value is dead now.
12651 CombineTo(LHS.getNode(), Load.getValue(0), Load.getValue(1));
12652 CombineTo(RHS.getNode(), Load.getValue(0), Load.getValue(1));
12703 // Turn "(a cond b) ? 1.0f : 2.0f" into "load (tmp + ((a cond b) ? 0 : 4)"
12719 // extra load, they are likely around in registers for other users.
13328 case ISD::LOAD:
13370 // L1 = load Token1, %52
13372 // L2 = load Token1, %52+8
13375 // L3 = load Token2, %53
13377 // L4 = load Token2, %53+8
