Bug type: crash
Component: LoopVectorize
Bug location: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp: LoopVectorizationPlanner::executePlan
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp: VPlanTransforms::handleUncountableEarlyExit
VPlanTransforms::materializeLiveInBroadcasts
Issue title: Assertion `!NodePtr->isKnownSentinel()' failed.
Labels: vectorizers
crash-on-valid
Reported by: TatyanaDoubts

To reproduce run opt with the test below, -passes=loop-vectorize
```
; ModuleID = './reduced.ll'
source_filename = "./reduced.ll"
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128-ni:1-p2:32:8:8:32-ni:2"
target triple = "x86_64-unknown-linux-gnu"

define void @wombat() gc "statepoint-example" personality ptr null {
bb:
  %invoke = invoke i32 null(ptr addrspace(1) null, i32 0, i32 0, i32 0)
          to label %bb1 unwind label %bb5

bb1:                                              ; preds = %bb1, %bb
  %phi = phi i64 [ %add, %bb1 ], [ 0, %bb ]
  %phi2 = phi i32 [ %add3, %bb1 ], [ 0, %bb ]
  %add = add i64 %phi, 1
  %sitofp = sitofp i32 %phi2 to double
  store double %sitofp, ptr addrspace(1) null, align 8
  %add3 = add i32 %phi2, %invoke
  %icmp = icmp ult i64 %phi, 161
  br i1 %icmp, label %bb1, label %bb4

bb4:                                              ; preds = %bb1
  ret void

bb5:                                              ; preds = %bb
  %landingpad = landingpad { ptr, i32 }
          cleanup
  ret void
}
```
Reproducer: https://godbolt.org/z/3do96adnn

Stack dump:
```
opt: /root/llvm-project/llvm/include/llvm/ADT/ilist_iterator.h:322: llvm::ilist_iterator_w_bits<OptionsT, IsReverse, IsConst>::reference llvm::ilist_iterator_w_bits<OptionsT, IsReverse, IsConst>::operator*() const [with OptionsT = llvm::ilist_detail::node_options<llvm::Instruction, true, false, void, true, llvm::BasicBlock>; bool IsReverse = false; bool IsConst = false; llvm::ilist_iterator_w_bits<OptionsT, IsReverse, IsConst>::reference = llvm::Instruction&]: Assertion `!NodePtr->isKnownSentinel()' failed.
PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace.
Stack dump:
0.	Program arguments: /opt/compiler-explorer/clang-assertions-trunk/bin/opt -o /app/output.s -S -passes=loop-vectorize <source>
1.	Running pass "function(loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only;>)" on module "<source>"
2.	Running pass "loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only;>" on function "wombat"
 #0 0x0000000005452b98 llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x5452b98)
 #1 0x0000000005450554 SignalHandler(int, siginfo_t*, void*) Signals.cpp:0:0
 #2 0x00007b16e6242520 (/lib/x86_64-linux-gnu/libc.so.6+0x42520)
 #3 0x00007b16e62969fc pthread_kill (/lib/x86_64-linux-gnu/libc.so.6+0x969fc)
 #4 0x00007b16e6242476 gsignal (/lib/x86_64-linux-gnu/libc.so.6+0x42476)
 #5 0x00007b16e62287f3 abort (/lib/x86_64-linux-gnu/libc.so.6+0x287f3)
 #6 0x00007b16e622871b (/lib/x86_64-linux-gnu/libc.so.6+0x2871b)
 #7 0x00007b16e6239e96 (/lib/x86_64-linux-gnu/libc.so.6+0x39e96)
 #8 0x00000000044f6c23 (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x44f6c23)
 #9 0x000000000450ebb3 llvm::VPTransformState::get(llvm::VPValue*, bool) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x450ebb3)
#10 0x0000000004543138 llvm::VPInstruction::generate(llvm::VPTransformState&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x4543138)
#11 0x0000000004544ba4 llvm::VPInstruction::execute(llvm::VPTransformState&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x4544ba4)
#12 0x00000000044f82e7 llvm::VPBasicBlock::executeRecipes(llvm::VPTransformState*, llvm::BasicBlock*) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x44f82e7)
#13 0x00000000045038f9 llvm::VPIRBasicBlock::execute(llvm::VPTransformState*) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x45038f9)
#14 0x0000000004510249 llvm::VPlan::execute(llvm::VPTransformState*) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x4510249)
#15 0x00000000043cccce llvm::LoopVectorizationPlanner::executePlan(llvm::ElementCount, unsigned int, llvm::VPlan&, llvm::InnerLoopVectorizer&, llvm::DominatorTree*, bool, llvm::DenseMap<llvm::SCEV const*, llvm::Value*, llvm::DenseMapInfo<llvm::SCEV const*, void>, llvm::detail::DenseMapPair<llvm::SCEV const*, llvm::Value*>> const*) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x43cccce)
#16 0x00000000043e15d8 llvm::LoopVectorizePass::processLoop(llvm::Loop*) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x43e15d8)
#17 0x00000000043e2f49 llvm::LoopVectorizePass::runImpl(llvm::Function&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x43e2f49)
#18 0x00000000043e35c3 llvm::LoopVectorizePass::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x43e35c3)
#19 0x000000000318aaae llvm::detail::PassModel<llvm::Function, llvm::LoopVectorizePass, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x318aaae)
#20 0x000000000524cd90 llvm::PassManager<llvm::Function, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x524cd90)
#21 0x0000000000e6f54e llvm::detail::PassModel<llvm::Function, llvm::PassManager<llvm::Function, llvm::AnalysisManager<llvm::Function>>, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0xe6f54e)
#22 0x000000000524b6ab llvm::ModuleToFunctionPassAdaptor::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x524b6ab)
#23 0x0000000000e6e83e llvm::detail::PassModel<llvm::Module, llvm::ModuleToFunctionPassAdaptor, llvm::AnalysisManager<llvm::Module>>::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0xe6e83e)
#24 0x000000000524b0b0 llvm::PassManager<llvm::Module, llvm::AnalysisManager<llvm::Module>>::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x524b0b0)
#25 0x0000000000935a3a llvm::runPassPipeline(llvm::StringRef, llvm::Module&, llvm::TargetMachine*, llvm::TargetLibraryInfoImpl*, llvm::ToolOutputFile*, llvm::ToolOutputFile*, llvm::ToolOutputFile*, llvm::StringRef, llvm::ArrayRef<llvm::PassPlugin>, llvm::ArrayRef<std::function<void (llvm::PassBuilder&)>>, llvm::opt_tool::OutputKind, llvm::opt_tool::VerifierKind, bool, bool, bool, bool, bool, bool, bool) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x935a3a)
#26 0x00000000009290ac optMain (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x9290ac)
#27 0x00007b16e6229d90 (/lib/x86_64-linux-gnu/libc.so.6+0x29d90)
#28 0x00007b16e6229e40 __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x29e40)
#29 0x0000000000920985 _start (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x920985)
Program terminated with signal: SIGSEGV
Compiler returned: 139
```

Comments: author: fhahn
body: looking 

Fix patch:
commit 62994c3291239f3d11f958d002319b2bd6eddbc9
Author: Florian Hahn <flo@fhahn.com>
Date:   Wed Mar 12 22:03:18 2025 +0000

    [VPlan] Also introduce explicit broadcasts for values from entry VPBB.
    
    Update and generalize materializeBroadcasts to also introduce explicit
    broadcasts for VPValues defined in the Plans Entry block.
    
    This fixes a crash when trying to insert the broadcasts generated by
    VPTransformState::get after the generating instruction, which isn't
    possible after invoke instructions.
    
    Fixes https://github.com/llvm/llvm-project/issues/128838.

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bab2c6efd403..02bacde3f60a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7691,11 +7691,11 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
       ((VectorizingEpilogue && ExpandedSCEVs) ||
        (!VectorizingEpilogue && !ExpandedSCEVs)) &&
       "expanded SCEVs to reuse can only be used during epilogue vectorization");
-  VPlanTransforms::materializeLiveInBroadcasts(BestVPlan);
   // TODO: Move to VPlan transform stage once the transition to the VPlan-based
   // cost model is complete for better cost estimates.
   VPlanTransforms::runPass(VPlanTransforms::unrollByUF, BestVPlan, BestUF,
                            OrigLoop->getHeader()->getContext());
+  VPlanTransforms::materializeBroadcasts(BestVPlan);
   VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
   VPlanTransforms::simplifyRecipes(BestVPlan, *Legal->getWidestInductionType());
   VPlanTransforms::removeDeadRecipes(BestVPlan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ecf52673480e..f23795f87d46 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2180,7 +2180,7 @@ void VPlanTransforms::handleUncountableEarlyExit(
   LatchExitingBranch->eraseFromParent();
 }
 
-void VPlanTransforms::materializeLiveInBroadcasts(VPlan &Plan) {
+void VPlanTransforms::materializeBroadcasts(VPlan &Plan) {
   if (Plan.hasScalarVFOnly())
     return;
 
@@ -2189,18 +2189,25 @@ void VPlanTransforms::materializeLiveInBroadcasts(VPlan &Plan) {
   VPDT.recalculate(Plan);
 #endif
 
+  SmallVector<VPValue *> VPValues;
+  append_range(VPValues, Plan.getLiveIns());
+  for (VPRecipeBase &R : *Plan.getEntry())
+    append_range(VPValues, R.definedValues());
+
   auto *VectorPreheader = Plan.getVectorPreheader();
-  for (VPValue *LiveIn : Plan.getLiveIns()) {
-    if (all_of(LiveIn->users(),
-               [LiveIn](VPUser *U) { return U->usesScalars(LiveIn); }) ||
-        !LiveIn->getLiveInIRValue() ||
-        isa<Constant>(LiveIn->getLiveInIRValue()))
+  for (VPValue *VPV : VPValues) {
+    if (all_of(VPV->users(),
+               [VPV](VPUser *U) { return U->usesScalars(VPV); }) ||
+        (VPV->isLiveIn() &&
+         (!VPV->getLiveInIRValue() || isa<Constant>(VPV->getLiveInIRValue()))))
       continue;
 
     // Add explicit broadcast at the insert point that dominates all users.
     VPBasicBlock *HoistBlock = VectorPreheader;
     VPBasicBlock::iterator HoistPoint = VectorPreheader->end();
-    for (VPUser *User : LiveIn->users()) {
+    for (VPUser *User : VPV->users()) {
+      if (User->usesScalars(VPV))
+        continue;
       if (cast<VPRecipeBase>(User)->getParent() == VectorPreheader)
         HoistPoint = HoistBlock->begin();
       else
@@ -2210,10 +2217,10 @@ void VPlanTransforms::materializeLiveInBroadcasts(VPlan &Plan) {
     }
 
     VPBuilder Builder(cast<VPBasicBlock>(HoistBlock), HoistPoint);
-    auto *Broadcast = Builder.createNaryOp(VPInstruction::Broadcast, {LiveIn});
-    LiveIn->replaceUsesWithIf(
-        Broadcast, [LiveIn, Broadcast](VPUser &U, unsigned Idx) {
-          return Broadcast != &U && !U.usesScalars(LiveIn);
-        });
+    auto *Broadcast = Builder.createNaryOp(VPInstruction::Broadcast, {VPV});
+    VPV->replaceUsesWithIf(Broadcast,
+                           [VPV, Broadcast](VPUser &U, unsigned Idx) {
+                             return Broadcast != &U && !U.usesScalars(VPV);
+                           });
   }
 }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 8a9a81e00fe2..2ec07eea1ecc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -189,8 +189,8 @@ struct VPlanTransforms {
   optimizeInductionExitUsers(VPlan &Plan,
                              DenseMap<VPValue *, VPValue *> &EndValues);
 
-  /// Add explicit broadcasts for live-ins used as vectors.
-  static void materializeLiveInBroadcasts(VPlan &Plan);
+  /// Add explicit broadcasts for live-ins and VPValues defined in \p Plan's entry block if they are used as vectors.
+  static void materializeBroadcasts(VPlan &Plan);
 };
 
 } // namespace llvm


Regression tests:
file: llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll
commands: opt < %s -passes=loop-vectorize -mattr=+v -force-vector-width=4 -scalable-vectorization=on -S 2>&1
tests: test_name: uniform_store_i1
test_body: target triple = "riscv64-linux-gnu"

define void @uniform_store_i1(ptr noalias %dst, ptr noalias %start, i64 %N) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %first.sroa = phi ptr [ %incdec.ptr, %for.body ], [ %start, %entry ]
  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
  %iv.next = add i64 %iv, 1
  %incdec.ptr = getelementptr inbounds i64, ptr %first.sroa, i64 1
  %cmp.not = icmp eq ptr %incdec.ptr, %start
  store i1 %cmp.not, ptr %dst, align 1
  %cmp = icmp ult i64 %iv, %N
  br i1 %cmp, label %for.body, label %end, !llvm.loop !0

end:                                              ; preds = %for.body
  ret void
}

!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}

file: llvm/test/Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll
commands: opt -passes=loop-vectorize -mcpu=znver2 -S %s
tests: test_name: test_induction_step_needs_expansion
test_body: target triple = "x86_64-unknown-linux-gnu"

define void @test_induction_step_needs_expansion(ptr noalias %j, ptr %k, i64 %l, i16 %off) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %p.09 = phi i16 [ 0, %entry ], [ %add, %loop ]
  %add = sub i16 %p.09, %off
  %arrayidx3 = getelementptr inbounds i16, ptr %k, i64 %iv
  store i16 %add, ptr %arrayidx3, align 2
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %l
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

file: llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
commands: opt -passes=loop-vectorize -force-vector-width=4 -S %s
tests: test_name: wide_add_induction_step_live_in
test_body: target triple = "x86_64-unknown-linux-gnu"

define i16 @wide_add_induction_step_live_in(ptr %dst, i64 %N, i16 %off) {
entry:
  %o.1 = add i16 %off, 2
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %iv.2 = phi i16 [ 0, %entry ], [ %add, %loop ]
  %add = add i16 %iv.2, %o.1
  %gep.dst = getelementptr inbounds i16, ptr %dst, i64 %iv
  store i16 %add, ptr %gep.dst, align 2
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %N
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret i16 %add
}

test_name: wide_sub_induction_step_live_in
test_body: target triple = "x86_64-unknown-linux-gnu"

define i16 @wide_sub_induction_step_live_in(ptr %dst, i64 %N, i16 %off) {
entry:
  %o.1 = add i16 %off, 2
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %iv.2 = phi i16 [ 0, %entry ], [ %sub, %loop ]
  %sub = sub i16 %iv.2, %o.1
  %gep.dst = getelementptr inbounds i16, ptr %dst, i64 %iv
  store i16 %sub, ptr %gep.dst, align 2
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %N
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret i16 %sub
}

file: llvm/test/Transforms/LoopVectorize/expand-scev-after-invoke.ll
commands: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s
tests: test_name: <module>
test_body: 
declare i32 @foo()

; Test case for https://github.com/llvm/llvm-project/issues/128838. Make sure
; we do not crash when expanding %step.
define void @test(ptr %dst) personality ptr null {
;
entry:
  %step = invoke i32 @foo()
  to label %loop unwind label %lpad

loop:
  %iv.1 = phi i64 [ 0, %entry ], [ %add, %loop ]
  %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop ]
  %add = add i64 %iv.1, 1
  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv.1
  store i32 %iv.2, ptr %gep, align 8
  %iv.2.next = add i32 %iv.2, %step
  %ec = icmp ult i64 %iv.1, 161
  br i1 %ec, label %loop, label %exit

exit:
  ret void

lpad:
  %landingpad = landingpad { ptr, i32 }
  cleanup
  ret void
}
file: llvm/test/Transforms/LoopVectorize/float-induction.ll
commands: opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-interleave=1 -force-vector-width=4 -S
opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-interleave=2 -force-vector-width=4 -S
opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-interleave=2 -force-vector-width=1 -S
opt < %s -passes=loop-vectorize,dce,simplifycfg,instcombine,simplifycfg -force-vector-interleave=1 -force-vector-width=2 -simplifycfg-require-and-preserve-domtree=1 -keep-loops=false -S
tests: test_name: fp_iv_loop1_fast_FMF
test_body: @fp_inc = external global float, align 4

define void @fp_iv_loop1_fast_FMF(float %init, ptr noalias captures(none) %A, i32 %N) {
entry:
  %cmp4 = icmp sgt i32 %N, 0
  br i1 %cmp4, label %for.body.lr.ph, label %for.end

for.body.lr.ph:                                   ; preds = %entry
  %fpinc = load float, ptr @fp_inc, align 4
  br label %for.body

for.body:                                         ; preds = %for.body, %for.body.lr.ph
  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
  %x.05 = phi float [ %init, %for.body.lr.ph ], [ %add, %for.body ]
  %arrayidx = getelementptr inbounds float, ptr %A, i64 %indvars.iv
  store float %x.05, ptr %arrayidx, align 4
  %add = fsub fast float %x.05, %fpinc
  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
  %exitcond = icmp eq i32 %lftr.wideiv, %N
  br i1 %exitcond, label %for.end.loopexit, label %for.body

for.end.loopexit:                                 ; preds = %for.body
  br label %for.end

for.end:                                          ; preds = %for.end.loopexit, %entry
  ret void
}

!llvm.module.flags = !{!0}

!0 = !{i32 2, !"Debug Info Version", i32 3}

test_name: fp_iv_loop3
test_body: @fp_inc = external global float, align 4

define void @fp_iv_loop3(float %init, ptr noalias captures(none) %A, ptr noalias captures(none) %B, ptr noalias captures(none) %C, i32 %N) {
entry:
  %cmp9 = icmp sgt i32 %N, 0
  br i1 %cmp9, label %for.body.lr.ph, label %for.end

for.body.lr.ph:                                   ; preds = %entry
  %0 = load float, ptr @fp_inc, align 4
  br label %for.body

for.body:                                         ; preds = %for.body, %for.body.lr.ph
  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
  %y.012 = phi float [ 0x3FB99999A0000000, %for.body.lr.ph ], [ %conv1, %for.body ]
  %x.011 = phi float [ %init, %for.body.lr.ph ], [ %add, %for.body ]
  %arrayidx = getelementptr inbounds float, ptr %A, i64 %indvars.iv
  store float %x.011, ptr %arrayidx, align 4
  %add = fadd fast float %x.011, %0
  %conv1 = fadd fast float %y.012, -5.000000e-01
  %add2 = fadd fast float %conv1, %add
  %arrayidx4 = getelementptr inbounds float, ptr %B, i64 %indvars.iv
  store float %add2, ptr %arrayidx4, align 4
  %arrayidx6 = getelementptr inbounds float, ptr %C, i64 %indvars.iv
  store float %conv1, ptr %arrayidx6, align 4
  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
  %exitcond = icmp eq i32 %lftr.wideiv, %N
  br i1 %exitcond, label %for.end.loopexit, label %for.body

for.end.loopexit:                                 ; preds = %for.body
  br label %for.end

for.end:                                          ; preds = %for.end.loopexit, %entry
  ret void
}

!llvm.module.flags = !{!0}

!0 = !{i32 2, !"Debug Info Version", i32 3}

test_name: fp_iv_loop1_reassoc_FMF
test_body: @fp_inc = external global float, align 4

define void @fp_iv_loop1_reassoc_FMF(float %init, ptr noalias captures(none) %A, i32 %N) {
entry:
  %cmp4 = icmp sgt i32 %N, 0
  br i1 %cmp4, label %for.body.lr.ph, label %for.end

for.body.lr.ph:                                   ; preds = %entry
  %fpinc = load float, ptr @fp_inc, align 4
  br label %for.body

for.body:                                         ; preds = %for.body, %for.body.lr.ph
  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
  %x.05 = phi float [ %init, %for.body.lr.ph ], [ %add, %for.body ]
  %arrayidx = getelementptr inbounds float, ptr %A, i64 %indvars.iv
  store float %x.05, ptr %arrayidx, align 4
  %add = fsub reassoc float %x.05, %fpinc
  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
  %exitcond = icmp eq i32 %lftr.wideiv, %N
  br i1 %exitcond, label %for.end.loopexit, label %for.body

for.end.loopexit:                                 ; preds = %for.body
  br label %for.end

for.end:                                          ; preds = %for.end.loopexit, %entry
  ret void
}

!llvm.module.flags = !{!0}

!0 = !{i32 2, !"Debug Info Version", i32 3}

file: llvm/test/Transforms/LoopVectorize/induction-step.ll
commands: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -S
tests: test_name: wide_add_induction_step_live_in
test_body: target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"

define void @wide_add_induction_step_live_in(ptr %dst, i64 %N, i16 %off) {
entry:
  %o.1 = add i16 %off, 2
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %iv.2 = phi i16 [ 0, %entry ], [ %add, %loop ]
  %add = add i16 %iv.2, %o.1
  %gep.dst = getelementptr inbounds i16, ptr %dst, i64 %iv
  store i16 %add, ptr %gep.dst, align 2
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %N
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

file: llvm/test/Transforms/LoopVectorize/induction.ll
commands: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S
opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=1 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S
opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=2 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S
opt < %s -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S
opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=2 -force-vector-width=4 -force-widen-divrem-via-safe-divisor=0 -enable-interleaved-mem-accesses -S
tests: test_name: test_optimized_cast_induction_feeding_first_order_recurrence
test_body: target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"

define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n, i32 %step, ptr %ptr) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %for = phi i32 [ 0, %entry ], [ %iv.2.conv, %loop ]
  %iv.1 = phi i64 [ 0, %entry ], [ %iv.1.next, %loop ]
  %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop ]
  %iv.2.ext = shl i32 %iv.2, 24
  %iv.2.conv = ashr exact i32 %iv.2.ext, 24
  %gep = getelementptr inbounds i32, ptr %ptr, i64 %iv.1
  store i32 %for, ptr %gep, align 4
  %iv.2.next = add nsw i32 %iv.2.conv, %step
  %iv.1.next = add nuw nsw i64 %iv.1, 1
  %exitcond = icmp eq i64 %iv.1.next, %n
  br i1 %exitcond, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}