Bug type: crash
Component: LoopVectorize
Bug location: llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp: narrowToSingleScalarRecipes
Issue title: [LV] Assert "Expected at least one loop-variant operand"
Labels: vectorizers
crash-on-valid
Reported by: sjoerdmeijer

This IR:

```
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
target triple = "aarch64-unknown-linux-gnu"

define i16 @_Z4testsisjyjybtjjxbxtthiPjPbS_PA25_A25_yS_S3_(ptr %arr_8, i64 %indvars.iv, i1 %tobool204.not) #0 {
entry:
  br label %for.cond57.preheader.us

for.cond57.preheader.us:                          ; preds = %for.cond57.preheader.us, %entry
  %indvars.iv696 = phi i64 [ %indvars.iv.next697, %for.cond57.preheader.us ], [ 0, %entry ]
  %arrayidx207.us571 = getelementptr i32, ptr %arr_8, i64 %indvars.iv
  %cond223.in.us572 = select i1 %tobool204.not, ptr null, ptr %arrayidx207.us571
  %cond223.us573 = load i32, ptr %cond223.in.us572, align 4
  %0 = trunc i32 %cond223.us573 to i16
  %indvars.iv.next697 = add i64 %indvars.iv696, 1
  %1 = icmp ult i64 %indvars.iv696, %indvars.iv
  br i1 %1, label %for.cond57.preheader.us, label %for.body62.us559.split, !llvm.loop !0

for.body62.us559.split:                           ; preds = %for.cond57.preheader.us
  ret i16 %0
}

attributes #0 = { "target-cpu"="grace" }

!0 = distinct !{!0, !1, !2, !3}
!1 = !{!"llvm.loop.mustprogress"}
!2 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
!3 = !{!"llvm.loop.vectorize.enable", i1 true}
```

triggers this assert:

```
opt: /root/llvm-project/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp:2552: virtual void llvm::VPWidenGEPRecipe::execute(llvm::VPTransformState&): Assertion `any_of(operands(), [](VPValue *Op) { return !Op->isDefinedOutsideLoopRegions(); }) && "Expected at least one loop-variant operand"' failed.
PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace and instructions to reproduce the bug.
Stack dump:
0.	Program arguments: /opt/compiler-explorer/clang-assertions-trunk/bin/opt -o /app/output.s -S -S -passes=loop-vectorize <source>
1.	Running pass "function(loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only;>)" on module "<source>"
2.	Running pass "loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only;>" on function "_Z4testsisjyjybtjjxbxtthiPjPbS_PA25_A25_yS_S3_"
 #0 0x000000000598ad38 llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x598ad38)
 #1 0x0000000005987be4 SignalHandler(int, siginfo_t*, void*) Signals.cpp:0:0
 #2 0x00007c4e84e42520 (/lib/x86_64-linux-gnu/libc.so.6+0x42520)
 #3 0x00007c4e84e969fc pthread_kill (/lib/x86_64-linux-gnu/libc.so.6+0x969fc)
 #4 0x00007c4e84e42476 gsignal (/lib/x86_64-linux-gnu/libc.so.6+0x42476)
 #5 0x00007c4e84e287f3 abort (/lib/x86_64-linux-gnu/libc.so.6+0x287f3)
 #6 0x00007c4e84e2871b (/lib/x86_64-linux-gnu/libc.so.6+0x2871b)
 #7 0x00007c4e84e39e96 (/lib/x86_64-linux-gnu/libc.so.6+0x39e96)
 #8 0x000000000388fda1 (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x388fda1)
 #9 0x000000000382fae3 llvm::VPBasicBlock::executeRecipes(llvm::VPTransformState*, llvm::BasicBlock*) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x382fae3)
#10 0x0000000003846025 llvm::VPBasicBlock::execute(llvm::VPTransformState*) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x3846025)
#11 0x000000000384b952 llvm::VPlan::execute(llvm::VPTransformState*) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x384b952)
#12 0x00000000036c1a93 llvm::LoopVectorizationPlanner::executePlan(llvm::ElementCount, unsigned int, llvm::VPlan&, llvm::InnerLoopVectorizer&, llvm::DominatorTree*, bool) (/opt/compiler-explorer/clang-assertions-trunk/bin/opt+0x36c1a93)
```

See also: https://godbolt.org/z/hnc9Wjxe4



Comments: author: sjoerdmeijer
body: CC: @fhahn 
author: fhahn
body: Should be fixed by https://github.com/llvm/llvm-project/pull/168246, plan to land it tomorrow

Fix patch:
commit b76089c7f3d6593d2e2c83db7dbf4965b656bd8c
Author: Florian Hahn <flo@fhahn.com>
Date:   Fri Nov 28 10:26:27 2025 +0000

    [VPlan] Skip uses-scalars restriction if one of ops needs broadcast. (#168246)
    
    Update the logic in narrowToSingleScalar to allow narrowing even if not
    all users use scalars, if at least one of the operands already needs
    broadcasting.
    
    In that case, there won't be any additional broadcasts introduced. This
    should allow removing the special handling for stores, which can
    introduce additional broadcasts currently.
    
    Fixes https://github.com/llvm/llvm-project/issues/169668.
    
    PR: https://github.com/llvm/llvm-project/pull/168246

diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 67aca48bcaf8..b12f8ccc73c7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1522,32 +1522,36 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
         continue;
       }
 
-      // Skip recipes that aren't single scalars or don't have only their
-      // scalar results used. In the latter case, we would introduce extra
-      // broadcasts.
-      if (!vputils::isSingleScalar(RepOrWidenR) ||
-          !all_of(RepOrWidenR->users(), [RepOrWidenR](const VPUser *U) {
-            if (auto *Store = dyn_cast<VPWidenStoreRecipe>(U)) {
-              // VPWidenStore doesn't have users, and stores are always
-              // profitable to widen: hence, permitting address and mask
-              // operands, and single-scalar stored values is an important leaf
-              // condition. The assert must hold as we checked the RepOrWidenR
-              // operand against vputils::isSingleScalar.
-              assert(RepOrWidenR != Store->getStoredValue() ||
-                     vputils::isSingleScalar(Store->getStoredValue()));
-              (void)Store;
-              return true;
-            }
-
-            if (auto *VPI = dyn_cast<VPInstruction>(U)) {
-              unsigned Opcode = VPI->getOpcode();
-              if (Opcode == VPInstruction::ExtractLastElement ||
-                  Opcode == VPInstruction::ExtractLastLanePerPart ||
-                  Opcode == VPInstruction::ExtractPenultimateElement)
-                return true;
-            }
-
-            return U->usesScalars(RepOrWidenR);
+      // Skip recipes that aren't single scalars.
+      if (!vputils::isSingleScalar(RepOrWidenR))
+        continue;
+
+      // Skip recipes for which conversion to single-scalar does introduce
+      // additional broadcasts. No extra broadcasts are needed, if either only
+      // the scalars of the recipe are used, or at least one of the operands
+      // would require a broadcast. In the latter case, the single-scalar may
+      // need to be broadcasted, but another broadcast is removed.
+      if (!all_of(RepOrWidenR->users(),
+                  [RepOrWidenR](const VPUser *U) {
+                    if (auto *VPI = dyn_cast<VPInstruction>(U)) {
+                      unsigned Opcode = VPI->getOpcode();
+                      if (Opcode == VPInstruction::ExtractLastElement ||
+                          Opcode == VPInstruction::ExtractLastLanePerPart ||
+                          Opcode == VPInstruction::ExtractPenultimateElement)
+                        return true;
+                    }
+
+                    return U->usesScalars(RepOrWidenR);
+                  }) &&
+          none_of(RepOrWidenR->operands(), [RepOrWidenR](VPValue *Op) {
+            if (Op->getSingleUser() != RepOrWidenR)
+              return false;
+            // Non-constant live-ins require broadcasts, while constants do not
+            // need explicit broadcasts.
+            bool LiveInNeedsBroadcast =
+                Op->isLiveIn() && !isa<Constant>(Op->getLiveInIRValue());
+            auto *OpR = dyn_cast<VPReplicateRecipe>(Op);
+            return LiveInNeedsBroadcast || (OpR && OpR->isSingleScalar());
           }))
         continue;
 


Regression tests:
file: llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
commands: opt -p loop-vectorize -S %s
opt -p loop-vectorize  -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S %s
tests: test_name: multiple_exit_conditions
test_body: target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
target triple = "arm64-apple-macosx14.0.0"

define void @multiple_exit_conditions(ptr %src, ptr noalias %dst) #0 {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %l = load i16, ptr %src, align 2
  %o = or i16 %l, 1
  %conv = uitofp i16 %o to double
  store double %conv, ptr %ptr.iv, align 8
  %iv.next = add nsw i64 %iv, 2
  %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 8
  %iv.clamp = and i64 %iv, 4294967294
  %ec = icmp eq i64 %iv.clamp, 512
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

attributes #0 = { "target-cpu"="neoverse-512tvb" }

file: llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
commands: opt -p loop-vectorize -S %s
opt -p loop-vectorize -mcpu=neoverse-v1 -S %s
opt -p loop-vectorize -mcpu=neoverse-v1 -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S %s
tests: test_name: chained_recurrences
test_body: target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
target triple = "arm64-apple-macosx14.0.0"

define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2) #0 {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %0 = phi i32 [ 0, %entry ], [ %3, %loop ]
  %1 = phi i32 [ 0, %entry ], [ %0, %loop ]
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %sum.red = phi i32 [ 0, %entry ], [ %red.2, %loop ]
  %2 = add i64 %y, 1
  %gep.1 = getelementptr i32, ptr %src.1, i64 %2
  %3 = load i32, ptr %gep.1, align 4
  %or3 = or i32 %1, %x
  %iv.next = add i64 %iv, 1
  %shr = lshr i32 %x, 1
  %4 = shl i32 %or3, 1
  %5 = or i32 %4, 2
  %shl19 = shl i32 %x, 1
  %6 = or i32 %shr, %shl19
  %7 = or i32 %6, %5
  %8 = or i32 %7, %x
  %or20 = or i32 %z, %x
  %not = and i32 %or20, 1
  %and = xor i32 %not, 1
  %idx.ext.1 = zext i32 %and to i64
  %gep.2 = getelementptr i32, ptr %src.2, i64 %idx.ext.1
  %9 = load i32, ptr %gep.2, align 4
  %shr24 = lshr i32 %8, 1
  %idx.ext.2 = zext i32 %shr24 to i64
  %gep.3 = getelementptr i32, ptr %src.2, i64 %idx.ext.2
  %10 = load i32, ptr %gep.3, align 4
  %red.1 = or i32 %9, %sum.red
  %red.2 = or i32 %red.1, %10
  %ec = icmp eq i64 %iv, %y
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret i32 %red.2
}

attributes #0 = { "target-features"="+sve" }

file: llvm/test/Transforms/LoopVectorize/AArch64/widen-gep-all-indices-invariant.ll
commands: opt -passes=loop-vectorize -mtriple=arm64-apple-macosx -S %s
tests: test_name: <module>
test_body: 
; Test case for https://github.com/llvm/llvm-project/issues/169668.
define i32 @gep_with_all_invariant_operands(ptr %src.0, ptr %src.1, i64 %n, i1 %cond) #0 {
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
  %gep = getelementptr i32, ptr %src.0, i64 %n
  %ptr = select i1 %cond, ptr %src.1, ptr %gep
  %val = load i32, ptr %ptr, align 4
  %iv.next = add i64 %iv, 1
  %cmp = icmp ult i64 %iv, %n
  br i1 %cmp, label %loop, label %exit, !llvm.loop !0

exit:
  ret i32 %val
}

attributes #0 = { "target-cpu"="neoverse-v2" }

!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.vectorize.enable", i1 true}
!2 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
;.
;.
file: llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
commands: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -S %s
tests: test_name: test
test_body: define void @test(ptr %p, i64 %a, i8 %b) {
entry:
  br label %for.cond

for.cond:                                         ; preds = %for.body, %entry
  %iv = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %add = add i32 %iv, 1
  %cmp.slt = icmp slt i32 %iv, 2
  %shl = shl i64 %a, 48
  %ashr = ashr i64 %shl, 52
  %trunc.i32 = trunc i64 %ashr to i32
  br i1 %cmp.slt, label %cond.false, label %for.body

cond.false:                                       ; preds = %for.cond
  %zext = zext i8 %b to i32
  br label %for.body

for.body:                                         ; preds = %cond.false, %for.cond
  %cond = phi i32 [ %trunc.i32, %for.cond ], [ %zext, %cond.false ]
  %shl.i32 = shl i32 %cond, 8
  %trunc = trunc i32 %shl.i32 to i8
  store i8 %trunc, ptr %p, align 1
  %cmp = icmp slt i32 %iv, 8
  br i1 %cmp, label %for.cond, label %exit

exit:                                             ; preds = %for.body
  ret void
}

file: llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
commands: opt < %s -passes=loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -S
tests: test_name: sdiv_by_zero
test_body: target triple = "x86_64-apple-macosx10.8.0"

define void @sdiv_by_zero(ptr noalias %src, ptr noalias %dst, i32 %d) #0 {
bb:
  br label %loop.header

loop.header:                                      ; preds = %loop.latch, %bb
  %iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %bb ]
  %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv
  %l = load i32, ptr %gep.src, align 4
  %icmp = icmp eq i32 %l, 0
  br i1 %icmp, label %loop.latch, label %then

then:                                             ; preds = %loop.header
  %sdiv = sdiv i32 %l, 0
  br label %loop.latch

loop.latch:                                       ; preds = %then, %loop.header
  %merge = phi i32 [ %sdiv, %then ], [ 0, %loop.header ]
  %gep.dst = getelementptr inbounds i32, ptr %dst, i64 %iv
  store i32 %merge, ptr %gep.dst, align 4
  %iv.next = add i64 %iv, 1
  %ec = icmp ult i64 %iv, 16
  br i1 %ec, label %loop.header, label %exit

exit:                                             ; preds = %loop.latch
  ret void
}

attributes #0 = { "target-cpu"="znver3" }

test_name: cost_ashr_with_op_known_invariant_via_scev
test_body: target triple = "x86_64-apple-macosx10.8.0"

define i32 @cost_ashr_with_op_known_invariant_via_scev(i8 %a) {
entry:
  %cmp.i = icmp eq i16 0, 0
  %conv.i = sext i16 0 to i32
  %conv5.i = sext i8 %a to i32
  br label %loop.header

loop.header:                                      ; preds = %loop.latch, %entry
  %iv = phi i8 [ 100, %entry ], [ %iv.next, %loop.latch ]
  br i1 %cmp.i, label %then, label %else

then:                                             ; preds = %else, %loop.header
  %p.1 = phi i32 [ %rem.i, %else ], [ 0, %loop.header ]
  %shr.i = ashr i32 %conv5.i, %p.1
  %tobool6.not.i = icmp eq i32 %shr.i, 0
  %sext.i = shl i32 %p.1, 24
  %0 = ashr exact i32 %sext.i, 24
  %1 = select i1 %tobool6.not.i, i32 %0, i32 0
  br label %loop.latch

else:                                             ; preds = %loop.header
  %rem.i = urem i32 -1, %conv.i
  %cmp3.i = icmp sgt i32 %rem.i, 1
  br i1 %cmp3.i, label %loop.latch, label %then

loop.latch:                                       ; preds = %else, %then
  %p.2 = phi i32 [ 0, %else ], [ %1, %then ]
  %iv.next = add i8 %iv, -1
  %ec = icmp eq i8 %iv.next, 0
  br i1 %ec, label %exit, label %loop.header

exit:                                             ; preds = %loop.latch
  ret i32 %p.2
}

test_name: test_predicated_udiv
test_body: target triple = "x86_64-apple-macosx10.8.0"

define i64 @test_predicated_udiv(i32 %d, i1 %c) #0 {
entry:
  br label %loop.header

loop.header:                                      ; preds = %loop.latch, %entry
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
  br i1 %c, label %loop.latch, label %then

then:                                             ; preds = %loop.header
  %call = tail call i32 @llvm.usub.sat.i32(i32 %iv, i32 1)
  %udiv = udiv i32 %call, %d
  %zext = zext i32 %udiv to i64
  br label %loop.latch

loop.latch:                                       ; preds = %then, %loop.header
  %merge = phi i64 [ %zext, %then ], [ 0, %loop.header ]
  %iv.next = add i32 %iv, 1
  %ec = icmp eq i32 %iv, 1000
  br i1 %ec, label %exit, label %loop.header

exit:                                             ; preds = %loop.latch
  ret i64 %merge
}

; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
declare i32 @llvm.usub.sat.i32(i32, i32) #1

attributes #0 = { "target-cpu"="znver3" }
attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }

file: llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
commands: opt < %s -passes=loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -S
tests: test_name: cost_loop_invariant_recipes
test_body: target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.8.0"

define i64 @cost_loop_invariant_recipes(i1 %x, i64 %y) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next.i.i.i, %loop ]
  %red = phi i64 [ 1, %entry ], [ %red.mul, %loop ]
  %not.x = xor i1 %x, true
  %ext = zext i1 %not.x to i64
  %shl = shl i64 %y, %ext
  %red.mul = mul i64 %shl, %red
  %iv.next.i.i.i = add i64 %iv, 1
  %ec = icmp eq i64 %iv, 1
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret i64 %red.mul
}

test_name: multi_exit
test_body: target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.8.0"

define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 {
entry:
  br label %loop

loop:                                             ; preds = %loop.latch, %entry
  %iv.1.wide = phi i64 [ 0, %entry ], [ %iv.1.next.wide, %loop.latch ]
  %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop.latch ]
  %ec.1 = icmp ult i64 %iv.1.wide, %A
  br i1 %ec.1, label %loop.latch, label %exit

loop.latch:                                       ; preds = %loop
  %gep.src.1 = getelementptr inbounds i64, ptr %src.1, i32 %iv.1
  %l.1 = load i64, ptr %gep.src.1, align 8
  %l.2 = load i64, ptr %src.2, align 8
  %cmp55.us = icmp eq i64 %l.1, 0
  %cmp.i.us = icmp ne i64 %l.2, 0
  %and = and i1 %cmp.i.us, %cmp55.us
  %ext = zext i1 %and to i8
  store i8 %ext, ptr %dst, align 1
  %iv.1.next = add i32 %iv.1, 1
  %iv.1.next.wide = zext i32 %iv.1.next to i64
  %ec.2 = icmp ult i64 %iv.1.next.wide, %B
  br i1 %ec.2, label %loop, label %exit

exit:                                             ; preds = %loop.latch, %loop
  ret void
}

attributes #0 = { "target-cpu"="penryn" }

file: llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
commands: opt -p loop-vectorize -mtriple=x86_64-linux-gnu -S %s
opt -p loop-vectorize -mtriple=i386-pc-linux-gnu -S %s
tests: test_name: test_load_used_by_other_load_scev_low_trip_count
test_body: declare i1 @cond()

define double @test_load_used_by_other_load_scev_low_trip_count(ptr %ptr.a, ptr %ptr.b, ptr %ptr.c) {
entry:
  br label %outer.loop

outer.loop:                                       ; preds = %inner.loop, %entry
  %accum = phi double [ 0.000000e+00, %entry ], [ %result, %inner.loop ]
  %cond = call i1 @cond()
  br i1 %cond, label %inner.loop, label %exit

inner.loop:                                       ; preds = %inner.loop, %outer.loop
  %iv = phi i64 [ 0, %outer.loop ], [ %iv.next, %inner.loop ]
  %accum.inner = phi double [ %accum, %outer.loop ], [ %mul1, %inner.loop ]
  %idx.plus1 = add i64 %iv, 1
  %gep.c = getelementptr i8, ptr %ptr.c, i64 %idx.plus1
  %gep.a.i64 = getelementptr i64, ptr %ptr.a, i64 %idx.plus1
  %load.idx = load i64, ptr %gep.a.i64, align 8
  %gep.b = getelementptr double, ptr %ptr.b, i64 %load.idx
  %load.a = load double, ptr %ptr.a, align 8
  %add1 = fadd double %load.a, 0.000000e+00
  %gep.c.offset = getelementptr i8, ptr %gep.c, i64 8
  %load.c = load double, ptr %gep.c.offset, align 8
  %mul1 = fmul double %add1, 0.000000e+00
  %mul2 = fmul double %load.c, 0.000000e+00
  %add2 = fadd double %mul2, 0.000000e+00
  %add3 = fadd double %add2, 1.000000e+00
  %load.b = load double, ptr %gep.b, align 8
  %div = fdiv double %load.b, %add3
  %result = fsub double %accum.inner, %div
  %iv.next = add i64 %iv, 1
  %exitcond = icmp eq i64 %iv, 1
  br i1 %exitcond, label %outer.loop, label %inner.loop

exit:                                             ; preds = %outer.loop
  ret double %accum
}

file: llvm/test/Transforms/LoopVectorize/cse-casts.ll
commands: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s
tests: test_name: preserve_flags_when_cloning_trunc
test_body: define i8 @preserve_flags_when_cloning_trunc(i8 %start, ptr noalias %src, ptr noalias %dst) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
  %red = phi i8 [ %red.next, %loop ], [ %start, %entry ]
  %l = load i32, ptr %src, align 4
  %cmp = icmp ne i32 %l, 0
  %cmp.ext = zext i1 %cmp to i64
  %cmp.trunc = trunc i64 %cmp.ext to i16
  %gep.dst = getelementptr i16, ptr %dst, i64 %iv
  store i16 %cmp.trunc, ptr %gep.dst, align 2
  %red.next = mul i8 %red, 3
  %iv.next = add i64 %iv, 1
  %ec = icmp ult i64 %iv, 416
  br i1 %ec, label %loop, label %exit

exit:                                             ; preds = %loop
  ret i8 %red.next
}

file: llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
commands: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output -S %s 2>&1
tests: test_name: test_chained_first_order_recurrences_4
test_body: define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
  %for.x = phi i64 [ %for.x.next, %loop ], [ 0, %entry ]
  %for.y = phi i32 [ %for.x.prev, %loop ], [ 0, %entry ]
  %iv.next = add i64 %iv, 1
  %gep = getelementptr i64, ptr %base, i64 %iv
  %for.x.prev = trunc i64 %for.x to i32
  %for.y.i64 = sext i32 %for.y to i64
  store i64 %for.y.i64, ptr %gep, align 4
  %for.x.next = mul i64 %x, 2
  %icmp = icmp ugt i64 %iv, 4096
  br i1 %icmp, label %ret, label %loop

ret:                                              ; preds = %loop
  ret i32 0
}

file: llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll
commands: opt -p loop-vectorize -force-vector-width=4 -S %s
opt -p loop-vectorize -force-vector-width=2 -force-vector-interleave=2 -S %s
tests: test_name: narrow_widen_store_user
test_body: define void @narrow_widen_store_user(i32 %x, ptr noalias %A, ptr noalias %B) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
  %gep.A = getelementptr i32, ptr %A, i32 %iv
  %gep.B = getelementptr i32, ptr %B, i32 %iv
  %wide.add = add i32 %x, 1
  %wide.mul = mul i32 %wide.add, 3
  store i32 %wide.add, ptr %gep.A, align 4
  store i32 %wide.mul, ptr %gep.B, align 4
  %iv.next = add i32 %iv, 1
  %ec = icmp ne i32 %iv.next, 1024
  br i1 %ec, label %loop, label %exit

exit:                                             ; preds = %loop
  ret void
}

file: llvm/test/Transforms/LoopVectorize/pr43166-fold-tail-by-masking.ll
commands: opt < %s -passes=loop-vectorize -force-vector-width=4 -S
tests: test_name: test2
test_body: define i64 @test2(i64 %y) {
entry:
  br label %for.body

for.body:                                         ; preds = %cond.end, %entry
  %i = phi i32 [ 0, %entry ], [ %inc, %cond.end ]
  %cmp = icmp eq i64 %y, 0
  br i1 %cmp, label %cond.end, label %cond.false

cond.false:                                       ; preds = %for.body
  br label %cond.end

cond.end:                                         ; preds = %cond.false, %for.body
  %cond = phi i64 [ 55, %cond.false ], [ 77, %for.body ]
  %inc = add nuw nsw i32 %i, 1
  %exitcond = icmp eq i32 %inc, 3
  br i1 %exitcond, label %for.cond.cleanup, label %for.body

for.cond.cleanup:                                 ; preds = %cond.end
  ret i64 %cond
}

test_name: test3
test_body: define i32 @test3(i64 %y) {
entry:
  br label %for.body

for.body:                                         ; preds = %cond.end, %entry
  %i = phi i32 [ 0, %entry ], [ %inc, %cond.end ]
  %cmp = icmp eq i64 %y, 0
  br i1 %cmp, label %cond.end, label %cond.false

cond.false:                                       ; preds = %for.body
  br label %cond.end

cond.end:                                         ; preds = %cond.false, %for.body
  %cond = phi i32 [ 55, %cond.false ], [ %i, %for.body ]
  %inc = add nuw nsw i32 %i, 1
  %exitcond = icmp eq i32 %inc, 3
  br i1 %exitcond, label %for.cond.cleanup, label %for.body

for.cond.cleanup:                                 ; preds = %cond.end
  ret i32 %cond
}

file: llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
commands: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S %s
tests: test_name: test
test_body: define i32 @test(i32 %a, i1 %c.1, i1 %c.2) {
bb:
  br label %loop.header

loop.header:                                      ; preds = %loop.latch, %bb
  %iv = phi i32 [ 6, %bb ], [ %iv.next, %loop.latch ]
  %v.2 = phi i32 [ 35902, %bb ], [ %p.2, %loop.latch ]
  br i1 %c.1, label %loop.latch, label %body.1

body.1:                                           ; preds = %loop.header
  %v.2.add = add i32 %v.2, 10
  br i1 %c.2, label %loop.latch, label %body.2

body.2:                                           ; preds = %body.1
  %add.1 = add i32 %v.2.add, 20
  %xor = xor i32 %a, 1
  %add.2 = add i32 %add.1, %xor
  br label %loop.latch

loop.latch:                                       ; preds = %body.2, %body.1, %loop.header
  %p.1 = phi i32 [ %iv, %loop.header ], [ 9, %body.1 ], [ 9, %body.2 ]
  %p.2 = phi i32 [ %v.2, %loop.header ], [ %v.2.add, %body.1 ], [ %add.2, %body.2 ]
  %iv.next = add nuw nsw i32 %iv, 1
  %ec = icmp ult i32 %iv, 181
  br i1 %ec, label %loop.header, label %exit

exit:                                             ; preds = %loop.latch
  %e.1 = phi i32 [ %p.1, %loop.latch ]
  %e.2 = phi i32 [ %p.2, %loop.latch ]
  %res = add i32 %e.1, %e.2
  ret i32 %res
}

file: llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
commands: opt -passes=loop-vectorize -force-vector-width=4 -S %s
tests: test_name: loop_invariant_srem
test_body: define void @loop_invariant_srem(ptr %p, i64 %a, i8 %b) {
entry:
  br label %loop.header

loop.header:                                      ; preds = %loop.latch, %entry
  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %iv.next = add i8 %iv, 1
  %cmp.slt = icmp slt i8 %iv, 2
  %shl = shl i64 %a, 48
  %ashr = ashr i64 %shl, 52
  %trunc.i32 = trunc i64 %ashr to i32
  br i1 %cmp.slt, label %cond.false, label %loop.latch

cond.false:                                       ; preds = %loop.header
  %zext = zext i8 %b to i32
  br label %loop.latch

loop.latch:                                       ; preds = %cond.false, %loop.header
  %cond = phi i32 [ %trunc.i32, %loop.header ], [ %zext, %cond.false ]
  %shl.i32 = shl i32 %cond, 8
  %trunc = trunc i32 %shl.i32 to i8
  %rem = srem i8 %iv, %trunc
  %gep.p.rem = getelementptr i32, ptr %p, i8 %rem
  store i32 4, ptr %gep.p.rem, align 4
  %ec = icmp eq i8 %iv, 8
  br i1 %ec, label %exit, label %loop.header

exit:                                             ; preds = %loop.latch
  ret void
}

test_name: loop_invariant_store
test_body: define void @loop_invariant_store(ptr %p, i64 %a, i8 %b) {
entry:
  br label %loop.header

loop.header:                                      ; preds = %loop.latch, %entry
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %iv.next = add i32 %iv, 1
  %cmp.slt = icmp slt i32 %iv, 2
  %shl = shl i64 %a, 48
  %ashr = ashr i64 %shl, 52
  %trunc.i32 = trunc i64 %ashr to i32
  br i1 %cmp.slt, label %cond.false, label %loop.latch

cond.false:                                       ; preds = %loop.header
  %zext = zext i8 %b to i32
  br label %loop.latch

loop.latch:                                       ; preds = %cond.false, %loop.header
  %cond = phi i32 [ %trunc.i32, %loop.header ], [ %zext, %cond.false ]
  %shl.i32 = shl i32 %cond, 8
  %trunc = trunc i32 %shl.i32 to i8
  store i8 %trunc, ptr %p, align 1
  %exitcond = icmp slt i32 %iv, 8
  br i1 %exitcond, label %loop.header, label %exit

exit:                                             ; preds = %loop.latch
  ret void
}

file: llvm/test/Transforms/LoopVectorize/reduction-inloop-uf4.ll
commands: opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-interleave=4 -force-vector-width=4 -prefer-inloop-reductions -S
tests: test_name: cond_rdx_pred
test_body: target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"

define i32 @cond_rdx_pred(i32 %cond, ptr noalias %a, i64 %N) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.inc, %entry
  %iv = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
  %sum = phi i32 [ %res, %for.inc ], [ 4, %entry ]
  %cmp1 = icmp sgt i32 %cond, 7
  br i1 %cmp1, label %if.then, label %for.inc

if.then:                                          ; preds = %for.body
  %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
  %load = load i32, ptr %arrayidx, align 4
  %mul = mul nsw i32 %load, %sum
  br label %for.inc

for.inc:                                          ; preds = %if.then, %for.body
  %res = phi i32 [ %mul, %if.then ], [ %sum, %for.body ]
  %inc = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %inc, %N
  br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0

for.end:                                          ; preds = %for.inc
  ret i32 %res
}

!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
!2 = !{!"llvm.loop.vectorize.enable", i1 true}

file: llvm/test/Transforms/LoopVectorize/trunc-shifts.ll
commands: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s
tests: test_name: test_ashr_const_shift_ops
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"

define void @test_ashr_const_shift_ops(ptr %dst, i32 %f) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
  %l = ashr i32 %f, 18
  %l.t = trunc i32 %l to i8
  %iv.ext = zext i8 %iv to i64
  %gep = getelementptr inbounds i8, ptr %dst, i64 %iv.ext
  store i8 %l.t, ptr %gep, align 8
  %iv.next = add i8 %iv, 1
  %conv = zext i8 %iv.next to i32
  %c = icmp ne i32 %conv, 100
  br i1 %c, label %loop, label %exit

exit:                                             ; preds = %loop
  ret void
}

test_name: test_pr47927_lshr_const_shift_ops
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"

define void @test_pr47927_lshr_const_shift_ops(ptr %dst, i32 %f) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
  %l = lshr i32 %f, 18
  %l.t = trunc i32 %l to i8
  %iv.ext = zext i8 %iv to i64
  %gep = getelementptr inbounds i8, ptr %dst, i64 %iv.ext
  store i8 %l.t, ptr %gep, align 8
  %iv.next = add i8 %iv, 1
  %conv = zext i8 %iv.next to i32
  %c = icmp ne i32 %conv, 100
  br i1 %c, label %loop, label %exit

exit:                                             ; preds = %loop
  ret void
}

test_name: test_shl_const_shift_ops
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"

define void @test_shl_const_shift_ops(ptr %dst, i32 %f) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
  %l = shl i32 %f, 18
  %l.t = trunc i32 %l to i8
  %iv.ext = zext i8 %iv to i64
  %gep = getelementptr inbounds i8, ptr %dst, i64 %iv.ext
  store i8 %l.t, ptr %gep, align 8
  %iv.next = add i8 %iv, 1
  %conv = zext i8 %iv.next to i32
  %c = icmp ne i32 %conv, 100
  br i1 %c, label %loop, label %exit

exit:                                             ; preds = %loop
  ret void
}