Home | History | Annotate | Download | only in LoopVectorize
      1 ; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -S | FileCheck %s
      2 ; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -instcombine -S | FileCheck %s --check-prefix=IND
      3 ; RUN: opt < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=2 -instcombine -S | FileCheck %s --check-prefix=UNROLL
      4 ; RUN: opt < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=2 -S | FileCheck %s --check-prefix=UNROLL-NO-IC
      5 ; RUN: opt < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=4 -enable-interleaved-mem-accesses -instcombine -S | FileCheck %s --check-prefix=INTERLEAVE
      6 
      7 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
      8 
      9 ; Make sure that we can handle multiple integer induction variables.
     10 ; CHECK-LABEL: @multi_int_induction(
     11 ; CHECK: vector.body:
     12 ; CHECK:  %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
     13 ; CHECK:  %[[VAR:.*]] = trunc i64 %index to i32
     14 ; CHECK:  %offset.idx = add i32 190, %[[VAR]]
     15 define void @multi_int_induction(i32* %A, i32 %N) {
     16 for.body.lr.ph:
     17   br label %for.body
     18 
     19 for.body:
     20   %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
     21   %count.09 = phi i32 [ 190, %for.body.lr.ph ], [ %inc, %for.body ]
     22   %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
     23   store i32 %count.09, i32* %arrayidx2, align 4
     24   %inc = add nsw i32 %count.09, 1
     25   %indvars.iv.next = add i64 %indvars.iv, 1
     26   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
     27   %exitcond = icmp ne i32 %lftr.wideiv, %N
     28   br i1 %exitcond, label %for.body, label %for.end
     29 
     30 for.end:
     31   ret void
     32 }
     33 
     34 ; Make sure we remove unneeded vectorization of induction variables.
     35 ; In order for instcombine to cleanup the vectorized induction variables that we
     36 ; create in the loop vectorizer we need to perform some form of redundancy
     37 ; elimination to get rid of multiple uses.
     38 
     39 ; IND-LABEL: scalar_use
     40 
     41 ; IND:     br label %vector.body
     42 ; IND:     vector.body:
     43 ;   Vectorized induction variable.
     44 ; IND-NOT:  insertelement <2 x i64>
     45 ; IND-NOT:  shufflevector <2 x i64>
     46 ; IND:     br {{.*}}, label %vector.body
     47 
     48 define void @scalar_use(float* %a, float %b, i64 %offset, i64 %offset2, i64 %n) {
     49 entry:
     50   br label %for.body
     51 
     52 for.body:
     53   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
     54   %ind.sum = add i64 %iv, %offset
     55   %arr.idx = getelementptr inbounds float, float* %a, i64 %ind.sum
     56   %l1 = load float, float* %arr.idx, align 4
     57   %ind.sum2 = add i64 %iv, %offset2
     58   %arr.idx2 = getelementptr inbounds float, float* %a, i64 %ind.sum2
     59   %l2 = load float, float* %arr.idx2, align 4
     60   %m = fmul fast float %b, %l2
     61   %ad = fadd fast float %l1, %m
     62   store float %ad, float* %arr.idx, align 4
     63   %iv.next = add nuw nsw i64 %iv, 1
     64   %exitcond = icmp eq i64 %iv.next, %n
     65   br i1 %exitcond, label %loopexit, label %for.body
     66 
     67 loopexit:
     68   ret void
     69 }
     70 
     71 ; Make sure we don't create a vector induction phi node that is unused.
     72 ; Scalarize the step vectors instead.
     73 ;
     74 ; for (int i = 0; i < n; ++i)
     75 ;   sum += a[i];
     76 ;
     77 ; CHECK-LABEL: @scalarize_induction_variable_01(
     78 ; CHECK: vector.body:
     79 ; CHECK:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
     80 ; CHECK:   %[[i0:.+]] = add i64 %index, 0
     81 ; CHECK:   %[[i1:.+]] = add i64 %index, 1
     82 ; CHECK:   getelementptr inbounds i64, i64* %a, i64 %[[i0]]
     83 ; CHECK:   getelementptr inbounds i64, i64* %a, i64 %[[i1]]
     84 ;
     85 ; UNROLL-NO-IC-LABEL: @scalarize_induction_variable_01(
     86 ; UNROLL-NO-IC: vector.body:
     87 ; UNROLL-NO-IC:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
     88 ; UNROLL-NO-IC:   %[[i0:.+]] = add i64 %index, 0
     89 ; UNROLL-NO-IC:   %[[i1:.+]] = add i64 %index, 1
     90 ; UNROLL-NO-IC:   %[[i2:.+]] = add i64 %index, 2
     91 ; UNROLL-NO-IC:   %[[i3:.+]] = add i64 %index, 3
     92 ; UNROLL-NO-IC:   getelementptr inbounds i64, i64* %a, i64 %[[i0]]
     93 ; UNROLL-NO-IC:   getelementptr inbounds i64, i64* %a, i64 %[[i1]]
     94 ; UNROLL-NO-IC:   getelementptr inbounds i64, i64* %a, i64 %[[i2]]
     95 ; UNROLL-NO-IC:   getelementptr inbounds i64, i64* %a, i64 %[[i3]]
     96 ;
     97 ; IND-LABEL: @scalarize_induction_variable_01(
     98 ; IND:     vector.body:
     99 ; IND:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
    100 ; IND-NOT:   add i64 {{.*}}, 2
    101 ; IND:       getelementptr inbounds i64, i64* %a, i64 %index
    102 ;
    103 ; UNROLL-LABEL: @scalarize_induction_variable_01(
    104 ; UNROLL:     vector.body:
    105 ; UNROLL:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
    106 ; UNROLL-NOT:   add i64 {{.*}}, 4
    107 ; UNROLL:       %[[g1:.+]] = getelementptr inbounds i64, i64* %a, i64 %index
    108 ; UNROLL:       getelementptr i64, i64* %[[g1]], i64 2
    109 
    110 define i64 @scalarize_induction_variable_01(i64 *%a, i64 %n) {
    111 entry:
    112   br label %for.body
    113 
    114 for.body:
    115   %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
    116   %sum = phi i64 [ %2, %for.body ], [ 0, %entry ]
    117   %0 = getelementptr inbounds i64, i64* %a, i64 %i
    118   %1 = load i64, i64* %0, align 8
    119   %2 = add i64 %1, %sum
    120   %i.next = add nuw nsw i64 %i, 1
    121   %cond = icmp slt i64 %i.next, %n
    122   br i1 %cond, label %for.body, label %for.end
    123 
    124 for.end:
    125   %3  = phi i64 [ %2, %for.body ]
    126   ret i64 %3
    127 }
    128 
    129 ; Make sure we scalarize the step vectors used for the pointer arithmetic. We
    130 ; can't easily simplify vectorized step vectors.
    131 ;
    132 ; float s = 0;
    133 ; for (int i ; 0; i < n; i += 8)
    134 ;   s += (a[i] + b[i] + 1.0f);
    135 ;
    136 ; CHECK-LABEL: @scalarize_induction_variable_02(
    137 ; CHECK: vector.body:
    138 ; CHECK:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
    139 ; CHECK:   %offset.idx = shl i64 %index, 3
    140 ; CHECK:   %[[i0:.+]] = add i64 %offset.idx, 0
    141 ; CHECK:   %[[i1:.+]] = add i64 %offset.idx, 8
    142 ; CHECK:   getelementptr inbounds float, float* %a, i64 %[[i0]]
    143 ; CHECK:   getelementptr inbounds float, float* %a, i64 %[[i1]]
    144 ; CHECK:   getelementptr inbounds float, float* %b, i64 %[[i0]]
    145 ; CHECK:   getelementptr inbounds float, float* %b, i64 %[[i1]]
    146 ;
    147 ; UNROLL-NO-IC-LABEL: @scalarize_induction_variable_02(
    148 ; UNROLL-NO-IC: vector.body:
    149 ; UNROLL-NO-IC:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
    150 ; UNROLL-NO-IC:   %offset.idx = shl i64 %index, 3
    151 ; UNROLL-NO-IC:   %[[i0:.+]] = add i64 %offset.idx, 0
    152 ; UNROLL-NO-IC:   %[[i1:.+]] = add i64 %offset.idx, 8
    153 ; UNROLL-NO-IC:   %[[i2:.+]] = add i64 %offset.idx, 16
    154 ; UNROLL-NO-IC:   %[[i3:.+]] = add i64 %offset.idx, 24
    155 ; UNROLL-NO-IC:   getelementptr inbounds float, float* %a, i64 %[[i0]]
    156 ; UNROLL-NO-IC:   getelementptr inbounds float, float* %a, i64 %[[i1]]
    157 ; UNROLL-NO-IC:   getelementptr inbounds float, float* %a, i64 %[[i2]]
    158 ; UNROLL-NO-IC:   getelementptr inbounds float, float* %a, i64 %[[i3]]
    159 ; UNROLL-NO-IC:   getelementptr inbounds float, float* %b, i64 %[[i0]]
    160 ; UNROLL-NO-IC:   getelementptr inbounds float, float* %b, i64 %[[i1]]
    161 ; UNROLL-NO-IC:   getelementptr inbounds float, float* %b, i64 %[[i2]]
    162 ; UNROLL-NO-IC:   getelementptr inbounds float, float* %b, i64 %[[i3]]
    163 ;
    164 ; IND-LABEL: @scalarize_induction_variable_02(
    165 ; IND: vector.body:
    166 ; IND:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
    167 ; IND:   %[[i0:.+]] = shl i64 %index, 3
    168 ; IND:   %[[i1:.+]] = or i64 %[[i0]], 8
    169 ; IND:   getelementptr inbounds float, float* %a, i64 %[[i0]]
    170 ; IND:   getelementptr inbounds float, float* %a, i64 %[[i1]]
    171 ;
    172 ; UNROLL-LABEL: @scalarize_induction_variable_02(
    173 ; UNROLL: vector.body:
    174 ; UNROLL:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
    175 ; UNROLL:   %[[i0:.+]] = shl i64 %index, 3
    176 ; UNROLL:   %[[i1:.+]] = or i64 %[[i0]], 8
    177 ; UNROLL:   %[[i2:.+]] = or i64 %[[i0]], 16
    178 ; UNROLL:   %[[i3:.+]] = or i64 %[[i0]], 24
    179 ; UNROLL:   getelementptr inbounds float, float* %a, i64 %[[i0]]
    180 ; UNROLL:   getelementptr inbounds float, float* %a, i64 %[[i1]]
    181 ; UNROLL:   getelementptr inbounds float, float* %a, i64 %[[i2]]
    182 ; UNROLL:   getelementptr inbounds float, float* %a, i64 %[[i3]]
    183 
    184 define float @scalarize_induction_variable_02(float* %a, float* %b, i64 %n) {
    185 entry:
    186   br label %for.body
    187 
    188 for.body:
    189   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
    190   %s = phi float [ 0.0, %entry ], [ %6, %for.body ]
    191   %0 = getelementptr inbounds float, float* %a, i64 %i
    192   %1 = load float, float* %0, align 4
    193   %2 = getelementptr inbounds float, float* %b, i64 %i
    194   %3 = load float, float* %2, align 4
    195   %4 = fadd fast float %s, 1.0
    196   %5 = fadd fast float %4, %1
    197   %6 = fadd fast float %5, %3
    198   %i.next = add nuw nsw i64 %i, 8
    199   %cond = icmp slt i64 %i.next, %n
    200   br i1 %cond, label %for.body, label %for.end
    201 
    202 for.end:
    203   %s.lcssa = phi float [ %6, %for.body ]
    204   ret float %s.lcssa
    205 }
    206 
    207 ; Make sure we scalarize the step vectors used for the pointer arithmetic. We
    208 ; can't easily simplify vectorized step vectors. (Interleaved accesses.)
    209 ;
    210 ; for (int i = 0; i < n; ++i)
    211 ;   a[i].f ^= y;
    212 ;
    213 ; INTERLEAVE-LABEL: @scalarize_induction_variable_03(
    214 ; INTERLEAVE: vector.body:
    215 ; INTERLEAVE:   %[[i0:.+]] = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
    216 ; INTERLEAVE:   %[[i1:.+]] = or i64 %[[i0]], 1
    217 ; INTERLEAVE:   %[[i2:.+]] = or i64 %[[i0]], 2
    218 ; INTERLEAVE:   %[[i3:.+]] = or i64 %[[i0]], 3
    219 ; INTERLEAVE:   %[[i4:.+]] = or i64 %[[i0]], 4
    220 ; INTERLEAVE:   %[[i5:.+]] = or i64 %[[i0]], 5
    221 ; INTERLEAVE:   %[[i6:.+]] = or i64 %[[i0]], 6
    222 ; INTERLEAVE:   %[[i7:.+]] = or i64 %[[i0]], 7
    223 ; INTERLEAVE:   getelementptr inbounds %pair, %pair* %p, i64 %[[i0]], i32 1
    224 ; INTERLEAVE:   getelementptr inbounds %pair, %pair* %p, i64 %[[i1]], i32 1
    225 ; INTERLEAVE:   getelementptr inbounds %pair, %pair* %p, i64 %[[i2]], i32 1
    226 ; INTERLEAVE:   getelementptr inbounds %pair, %pair* %p, i64 %[[i3]], i32 1
    227 ; INTERLEAVE:   getelementptr inbounds %pair, %pair* %p, i64 %[[i4]], i32 1
    228 ; INTERLEAVE:   getelementptr inbounds %pair, %pair* %p, i64 %[[i5]], i32 1
    229 ; INTERLEAVE:   getelementptr inbounds %pair, %pair* %p, i64 %[[i6]], i32 1
    230 ; INTERLEAVE:   getelementptr inbounds %pair, %pair* %p, i64 %[[i7]], i32 1
    231 
    232 %pair = type { i32, i32 }
    233 define void @scalarize_induction_variable_03(%pair *%p, i32 %y, i64 %n) {
    234 entry:
    235   br label %for.body
    236 
    237 for.body:
    238   %i  = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
    239   %f = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1
    240   %0 = load i32, i32* %f, align 8
    241   %1 = xor i32 %0, %y
    242   store i32 %1, i32* %f, align 8
    243   %i.next = add nuw nsw i64 %i, 1
    244   %cond = icmp slt i64 %i.next, %n
    245   br i1 %cond, label %for.body, label %for.end
    246 
    247 for.end:
    248   ret void
    249 }
    250 
    251 ; Make sure that the loop exit count computation does not overflow for i8 and
    252 ; i16. The exit count of these loops is i8/i16 max + 1. If we don't cast the
    253 ; induction variable to a bigger type the exit count computation will overflow
    254 ; to 0.
    255 ; PR17532
    256 
    257 ; CHECK-LABEL: i8_loop
    258 ; CHECK: icmp eq i32 {{.*}}, 256
    259 define i32 @i8_loop() nounwind readnone ssp uwtable {
    260   br label %1
    261 
    262 ; <label>:1                                       ; preds = %1, %0
    263   %a.0 = phi i32 [ 1, %0 ], [ %2, %1 ]
    264   %b.0 = phi i8 [ 0, %0 ], [ %3, %1 ]
    265   %2 = and i32 %a.0, 4
    266   %3 = add i8 %b.0, -1
    267   %4 = icmp eq i8 %3, 0
    268   br i1 %4, label %5, label %1
    269 
    270 ; <label>:5                                       ; preds = %1
    271   ret i32 %2
    272 }
    273 
    274 ; CHECK-LABEL: i16_loop
    275 ; CHECK: icmp eq i32 {{.*}}, 65536
    276 
    277 define i32 @i16_loop() nounwind readnone ssp uwtable {
    278   br label %1
    279 
    280 ; <label>:1                                       ; preds = %1, %0
    281   %a.0 = phi i32 [ 1, %0 ], [ %2, %1 ]
    282   %b.0 = phi i16 [ 0, %0 ], [ %3, %1 ]
    283   %2 = and i32 %a.0, 4
    284   %3 = add i16 %b.0, -1
    285   %4 = icmp eq i16 %3, 0
    286   br i1 %4, label %5, label %1
    287 
    288 ; <label>:5                                       ; preds = %1
    289   ret i32 %2
    290 }
    291 
    292 ; This loop has a backedge taken count of i32_max. We need to check for this
    293 ; condition and branch directly to the scalar loop.
    294 
    295 ; CHECK-LABEL: max_i32_backedgetaken
    296 ; CHECK:  br i1 true, label %scalar.ph, label %min.iters.checked
    297 
    298 ; CHECK: middle.block:
    299 ; CHECK:  %[[v9:.+]] = extractelement <2 x i32> %bin.rdx, i32 0
    300 ; CHECK: scalar.ph:
    301 ; CHECK:  %bc.resume.val = phi i32 [ 0, %middle.block ], [ 0, %[[v0:.+]] ]
    302 ; CHECK:  %bc.merge.rdx = phi i32 [ 1, %[[v0:.+]] ], [ 1, %min.iters.checked ], [ %[[v9]], %middle.block ]
    303 
    304 define i32 @max_i32_backedgetaken() nounwind readnone ssp uwtable {
    305 
    306   br label %1
    307 
    308 ; <label>:1                                       ; preds = %1, %0
    309   %a.0 = phi i32 [ 1, %0 ], [ %2, %1 ]
    310   %b.0 = phi i32 [ 0, %0 ], [ %3, %1 ]
    311   %2 = and i32 %a.0, 4
    312   %3 = add i32 %b.0, -1
    313   %4 = icmp eq i32 %3, 0
    314   br i1 %4, label %5, label %1
    315 
    316 ; <label>:5                                       ; preds = %1
    317   ret i32 %2
    318 }
    319 
    320 ; When generating the overflow check we must sure that the induction start value
    321 ; is defined before the branch to the scalar preheader.
    322 
    323 ; CHECK-LABEL: testoverflowcheck
    324 ; CHECK: entry
    325 ; CHECK: %[[LOAD:.*]] = load i8
    326 ; CHECK: br
    327 
    328 ; CHECK: scalar.ph
    329 ; CHECK: phi i8 [ %{{.*}}, %middle.block ], [ %[[LOAD]], %entry ]
    330 
    331 @e = global i8 1, align 1
    332 @d = common global i32 0, align 4
    333 @c = common global i32 0, align 4
    334 define i32 @testoverflowcheck() {
    335 entry:
    336   %.pr.i = load i8, i8* @e, align 1
    337   %0 = load i32, i32* @d, align 4
    338   %c.promoted.i = load i32, i32* @c, align 4
    339   br label %cond.end.i
    340 
    341 cond.end.i:
    342   %inc4.i = phi i8 [ %.pr.i, %entry ], [ %inc.i, %cond.end.i ]
    343   %and3.i = phi i32 [ %c.promoted.i, %entry ], [ %and.i, %cond.end.i ]
    344   %and.i = and i32 %0, %and3.i
    345   %inc.i = add i8 %inc4.i, 1
    346   %tobool.i = icmp eq i8 %inc.i, 0
    347   br i1 %tobool.i, label %loopexit, label %cond.end.i
    348 
    349 loopexit:
    350   ret i32 %and.i
    351 }
    352 
    353 ; The SCEV expression of %sphi is (zext i8 {%t,+,1}<%loop> to i32)
    354 ; In order to recognize %sphi as an induction PHI and vectorize this loop,
    355 ; we need to convert the SCEV expression into an AddRecExpr.
    356 ; The expression gets converted to {zext i8 %t to i32,+,1}.
    357 
    358 ; CHECK-LABEL: wrappingindvars1
    359 ; CHECK-LABEL: vector.scevcheck
    360 ; CHECK-LABEL: vector.ph
    361 ; CHECK: %[[START:.*]] = add <2 x i32> %{{.*}}, <i32 0, i32 1>
    362 ; CHECK-LABEL: vector.body
    363 ; CHECK: %[[PHI:.*]] = phi <2 x i32> [ %[[START]], %vector.ph ], [ %[[STEP:.*]], %vector.body ]
    364 ; CHECK: %[[STEP]] = add <2 x i32> %[[PHI]], <i32 2, i32 2>
    365 define void @wrappingindvars1(i8 %t, i32 %len, i32 *%A) {
    366  entry:
    367   %st = zext i8 %t to i16
    368   %ext = zext i8 %t to i32
    369   %ecmp = icmp ult i16 %st, 42
    370   br i1 %ecmp, label %loop, label %exit
    371 
    372  loop:
    373 
    374   %idx = phi i8 [ %t, %entry ], [ %idx.inc, %loop ]
    375   %idx.b = phi i32 [ 0, %entry ], [ %idx.b.inc, %loop ]
    376   %sphi = phi i32 [ %ext, %entry ], [%idx.inc.ext, %loop]
    377 
    378   %ptr = getelementptr inbounds i32, i32* %A, i8 %idx
    379   store i32 %sphi, i32* %ptr
    380 
    381   %idx.inc = add i8 %idx, 1
    382   %idx.inc.ext = zext i8 %idx.inc to i32
    383   %idx.b.inc = add nuw nsw i32 %idx.b, 1
    384 
    385   %c = icmp ult i32 %idx.b, %len
    386   br i1 %c, label %loop, label %exit
    387 
    388  exit:
    389   ret void
    390 }
    391 
    392 ; The SCEV expression of %sphi is (4 * (zext i8 {%t,+,1}<%loop> to i32))
    393 ; In order to recognize %sphi as an induction PHI and vectorize this loop,
    394 ; we need to convert the SCEV expression into an AddRecExpr.
    395 ; The expression gets converted to ({4 * (zext %t to i32),+,4}).
    396 ; CHECK-LABEL: wrappingindvars2
    397 ; CHECK-LABEL: vector.scevcheck
    398 ; CHECK-LABEL: vector.ph
    399 ; CHECK: %[[START:.*]] = add <2 x i32> %{{.*}}, <i32 0, i32 4>
    400 ; CHECK-LABEL: vector.body
    401 ; CHECK: %[[PHI:.*]] = phi <2 x i32> [ %[[START]], %vector.ph ], [ %[[STEP:.*]], %vector.body ]
    402 ; CHECK: %[[STEP]] = add <2 x i32> %[[PHI]], <i32 8, i32 8>
    403 define void @wrappingindvars2(i8 %t, i32 %len, i32 *%A) {
    404 
    405 entry:
    406   %st = zext i8 %t to i16
    407   %ext = zext i8 %t to i32
    408   %ext.mul = mul i32 %ext, 4
    409 
    410   %ecmp = icmp ult i16 %st, 42
    411   br i1 %ecmp, label %loop, label %exit
    412 
    413  loop:
    414 
    415   %idx = phi i8 [ %t, %entry ], [ %idx.inc, %loop ]
    416   %sphi = phi i32 [ %ext.mul, %entry ], [%mul, %loop]
    417   %idx.b = phi i32 [ 0, %entry ], [ %idx.b.inc, %loop ]
    418 
    419   %ptr = getelementptr inbounds i32, i32* %A, i8 %idx
    420   store i32 %sphi, i32* %ptr
    421 
    422   %idx.inc = add i8 %idx, 1
    423   %idx.inc.ext = zext i8 %idx.inc to i32
    424   %mul = mul i32 %idx.inc.ext, 4
    425   %idx.b.inc = add nuw nsw i32 %idx.b, 1
    426 
    427   %c = icmp ult i32 %idx.b, %len
    428   br i1 %c, label %loop, label %exit
    429 
    430  exit:
    431   ret void
    432 }
    433 
    434 ; Check that we generate vectorized IVs in the pre-header
    435 ; instead of widening the scalar IV inside the loop, when
    436 ; we know how to do that.
    437 ; IND-LABEL: veciv
    438 ; IND: vector.body:
    439 ; IND: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
    440 ; IND: %vec.ind = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %step.add, %vector.body ]
    441 ; IND: %step.add = add <2 x i32> %vec.ind, <i32 2, i32 2>
    442 ; IND: %index.next = add i32 %index, 2
    443 ; IND: %[[CMP:.*]] = icmp eq i32 %index.next
    444 ; IND: br i1 %[[CMP]]
    445 ; UNROLL-LABEL: veciv
    446 ; UNROLL: vector.body:
    447 ; UNROLL: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
    448 ; UNROLL: %vec.ind = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %step.add1, %vector.body ]
    449 ; UNROLL: %step.add = add <2 x i32> %vec.ind, <i32 2, i32 2>
    450 ; UNROLL: %step.add1 = add <2 x i32> %vec.ind, <i32 4, i32 4>
    451 ; UNROLL: %index.next = add i32 %index, 4
    452 ; UNROLL: %[[CMP:.*]] = icmp eq i32 %index.next
    453 ; UNROLL: br i1 %[[CMP]]
    454 define void @veciv(i32* nocapture %a, i32 %start, i32 %k) {
    455 for.body.preheader:
    456   br label %for.body
    457 
    458 for.body:
    459   %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
    460   %arrayidx = getelementptr inbounds i32, i32* %a, i32 %indvars.iv
    461   store i32 %indvars.iv, i32* %arrayidx, align 4
    462   %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
    463   %exitcond = icmp eq i32 %indvars.iv.next, %k
    464   br i1 %exitcond, label %exit, label %for.body
    465 
    466 exit:
    467   ret void
    468 }
    469 
    470 ; IND-LABEL: trunciv
    471 ; IND: vector.body:
    472 ; IND: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
    473 ; IND: %[[VECIND:.*]] = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %[[STEPADD:.*]], %vector.body ]
    474 ; IND: %[[STEPADD]] = add <2 x i32> %[[VECIND]], <i32 2, i32 2>
    475 ; IND: %index.next = add i64 %index, 2
    476 ; IND: %[[CMP:.*]] = icmp eq i64 %index.next
    477 ; IND: br i1 %[[CMP]]
    478 define void @trunciv(i32* nocapture %a, i32 %start, i64 %k) {
    479 for.body.preheader:
    480   br label %for.body
    481 
    482 for.body:
    483   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
    484   %trunc.iv = trunc i64 %indvars.iv to i32
    485   %arrayidx = getelementptr inbounds i32, i32* %a, i32 %trunc.iv
    486   store i32 %trunc.iv, i32* %arrayidx, align 4
    487   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
    488   %exitcond = icmp eq i64 %indvars.iv.next, %k
    489   br i1 %exitcond, label %exit, label %for.body
    490 
    491 exit:
    492   ret void
    493 }
    494 
    495 ; IND-LABEL: nonprimary
    496 ; IND-LABEL: vector.ph
    497 ; IND: %[[INSERT:.*]] = insertelement <2 x i32> undef, i32 %i, i32 0
    498 ; IND: %[[SPLAT:.*]] = shufflevector <2 x i32> %[[INSERT]], <2 x i32> undef, <2 x i32> zeroinitializer
    499 ; IND: %[[START:.*]] = add <2 x i32> %[[SPLAT]], <i32 0, i32 42>
    500 ; IND-LABEL: vector.body:
    501 ; IND: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
    502 ; IND: %vec.ind = phi <2 x i32> [ %[[START]], %vector.ph ], [ %step.add, %vector.body ]
    503 ; IND: %step.add = add <2 x i32> %vec.ind, <i32 84, i32 84>
    504 ; IND: %index.next = add i32 %index, 2
    505 ; IND: %[[CMP:.*]] = icmp eq i32 %index.next
    506 ; IND: br i1 %[[CMP]]
    507 ; UNROLL-LABEL: nonprimary
    508 ; UNROLL-LABEL: vector.ph
    509 ; UNROLL: %[[INSERT:.*]] = insertelement <2 x i32> undef, i32 %i, i32 0
    510 ; UNROLL: %[[SPLAT:.*]] = shufflevector <2 x i32> %[[INSERT]], <2 x i32> undef, <2 x i32> zeroinitializer
    511 ; UNROLL: %[[START:.*]] = add <2 x i32> %[[SPLAT]], <i32 0, i32 42>
    512 ; UNROLL-LABEL: vector.body:
    513 ; UNROLL: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
    514 ; UNROLL: %vec.ind = phi <2 x i32> [ %[[START]], %vector.ph ], [ %step.add1, %vector.body ]
    515 ; UNROLL: %step.add = add <2 x i32> %vec.ind, <i32 84, i32 84>
    516 ; UNROLL: %step.add1 = add <2 x i32> %vec.ind, <i32 168, i32 168>
    517 ; UNROLL: %index.next = add i32 %index, 4
    518 ; UNROLL: %[[CMP:.*]] = icmp eq i32 %index.next
    519 ; UNROLL: br i1 %[[CMP]]
    520 define void @nonprimary(i32* nocapture %a, i32 %start, i32 %i, i32 %k) {
    521 for.body.preheader:
    522   br label %for.body
    523 
    524 for.body:
    525   %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ %i, %for.body.preheader ]
    526   %arrayidx = getelementptr inbounds i32, i32* %a, i32 %indvars.iv
    527   store i32 %indvars.iv, i32* %arrayidx, align 4
    528   %indvars.iv.next = add nuw nsw i32 %indvars.iv, 42
    529   %exitcond = icmp eq i32 %indvars.iv.next, %k
    530   br i1 %exitcond, label %exit, label %for.body
    531 
    532 exit:
    533   ret void
    534 }
    535