Bug type: miscompilation
Component: LoopVectorize
Bug location: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp: LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp: VPlanTransforms::handleEarlyExits
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp: VPInstruction::computeCost
VPInstruction::generate
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp: VPlanTransforms::convertToConcreteRecipes
VPlanTransforms::handleUncountableEarlyExit
Issue title: wrong code at -O{s,2} with "-mllvm -force-vector-width=1 -mllvm -force-vector-interleave=2" on x86_64-linux-gnu
Labels: miscompilation
vectorizers
Reported by: zhendongsu

Compiler Explorer: https://godbolt.org/z/eEqc4oeET

Note:
- works: 20.1.0 and earlier
- fails: trunk, only at -O{s,2}
- perhaps related: https://github.com/llvm/llvm-project/issues/153946 

```
[516] % clangtk -v
clang version 22.0.0git (https://github.com/llvm/llvm-project.git fc6299065794d05bcab8f6bb9628441b3d071131)
Target: x86_64-unknown-linux-gnu
Thread model: posix
InstalledDir: /local/home/suz/suz-local/software/local/clang-trunk/bin
Build config: +assertions
Found candidate GCC installation: /usr/lib/gcc/x86_64-linux-gnu/11
Found candidate GCC installation: /usr/lib/gcc/x86_64-linux-gnu/12
Selected GCC installation: /usr/lib/gcc/x86_64-linux-gnu/12
Candidate multilib: .;@m64
Selected multilib: .;@m64
Found CUDA installation: /usr/local/cuda, version 12.1
[517] % 
[517] % clangtk -O2 small.c; ./a.out
[518] % 
[518] % clangtk -O2 -mllvm -force-vector-width=1 -mllvm -force-vector-interleave=2 small.c
[519] % ./a.out
Aborted
[520] % cat small.c
int a, b;
int f(int g) {
  a = 0;
  for (; a < 32; a++)
    if (g >> a & 1)
      return a;
  return 0;
}
int main() {
  if (f(b-2) != 1)
    __builtin_abort();
  return 0;
}
```

Comments: author: MacDue
body: This seems to be an issue with early-exit vectorization not handling a return value depending on the induction variable correctly. It looks like it just passes through the vectorized induction `vp<%4>`:

```
<x1> vector loop: {
  vector.body:
    EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
    vp<%4> = SCALAR-STEPS vp<%3>, ir<1>, vp<%0>
    CLONE ir<%0> = shl nuw ir<1>, vp<%4>
    CLONE ir<%1> = and ir<%0>, ir<%g>
    CLONE ir<%tobool.not> = icmp eq ir<%1>, ir<0>
    EMIT vp<%index.next> = add nuw vp<%3>, vp<%1>
    EMIT vp<%5> = not ir<%tobool.not>
    EMIT vp<%6> = any-of vp<%5>
    EMIT vp<%7> = icmp eq vp<%index.next>, vp<%2>
    EMIT vp<%8> = or vp<%6>, vp<%7>
    EMIT branch-on-cond vp<%8>
  No successors
}
Successor(s): middle.split

middle.split:
  EMIT branch-on-cond vp<%6>
Successor(s): vector.early.exit, middle.block

middle.block:
  EMIT vp<%cmp.n> = icmp eq ir<32>, vp<%2>
  EMIT branch-on-cond vp<%cmp.n>
Successor(s): ir-bb<return>, scalar.ph

vector.early.exit:
Successor(s): ir-bb<return>

ir-bb<return>:
  IR   %storemerge.lcssa = phi i32 [ 32, %for.inc ], [ %storemerge2, %for.body ] (extra operands: ir<32> from middle.block, vp<%4> from vector.early.exit)
  IR   %retval.0 = phi i32 [ 0, %for.inc ], [ %storemerge2, %for.body ] (extra operands: ir<0> from middle.block, vp<%4> from vector.early.exit)
  IR   store i32 %storemerge.lcssa, ptr @a, align 4, !tbaa !7
No successors
```

author: fhahn
body: Don't think this is related to https://github.com/llvm/llvm-project/issues/153946, it is specific to VF = 1. Will look into a fix

Fix patch:
commit c950a729749fc32a9431643d3d149bd833e01451
Author: Florian Hahn <flo@fhahn.com>
Date:   Mon Aug 25 21:42:14 2025 +0100

    [VPlan] Support scalar VF for ExtractLane and FirstActiveLane.
    
    Extend ExtractLane and FirstActiveLane to support scalable VFs. This
    allows correct handling when interleaving with VF = 1.
    
    Alive2 proofs:
     - Fixed codegen with this patch: https://alive2.llvm.org/ce/z/8Y5_Vc
       (verifies as correct)
     - Original codegen: https://alive2.llvm.org/ce/z/twdg3X (doesn't
       verify)
    
    Fixes https://github.com/llvm/llvm-project/issues/154967.

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a0f306c12754..cc99386a4e66 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8634,8 +8634,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
             return !CM.requiresScalarEpilogue(VF.isVector());
           },
           Range);
-  VPlanTransforms::handleEarlyExits(*Plan, Legal->hasUncountableEarlyExit(),
-                                    Range);
+  VPlanTransforms::handleEarlyExits(*Plan, Legal->hasUncountableEarlyExit());
   VPlanTransforms::addMiddleCheck(*Plan, RequiresScalarEpilogueCheck,
                                   CM.foldTailByMasking());
 
@@ -8926,7 +8925,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
       OrigLoop, *LI, Legal->getWidestInductionType(),
       getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
   VPlanTransforms::handleEarlyExits(*Plan,
-                                    /*HasUncountableExit*/ false, Range);
+                                    /*HasUncountableExit*/ false);
   VPlanTransforms::addMiddleCheck(*Plan, /*RequiresScalarEpilogue*/ true,
                                   /*TailFolded*/ false);
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 80b48de57b40..4a8b4b8d0484 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -538,8 +538,7 @@ VPlanTransforms::buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
 }
 
 void VPlanTransforms::handleEarlyExits(VPlan &Plan,
-                                       bool HasUncountableEarlyExit,
-                                       VFRange &Range) {
+                                       bool HasUncountableEarlyExit) {
   auto *MiddleVPBB = cast<VPBasicBlock>(
       Plan.getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
   auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
@@ -559,8 +558,7 @@ void VPlanTransforms::handleEarlyExits(VPlan &Plan,
         assert(!HandledUncountableEarlyExit &&
                "can handle exactly one uncountable early exit");
         handleUncountableEarlyExit(cast<VPBasicBlock>(Pred), EB, Plan,
-                                   cast<VPBasicBlock>(HeaderVPB), LatchVPBB,
-                                   Range);
+                                   cast<VPBasicBlock>(HeaderVPB), LatchVPBB);
         HandledUncountableEarlyExit = true;
       } else {
         for (VPRecipeBase &R : EB->phis())
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2f3ee1ff61d2..f804846f82cc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -919,8 +919,15 @@ Value *VPInstruction::generate(VPTransformState &State) {
     unsigned LastOpIdx = getNumOperands() - 1;
     Value *Res = nullptr;
     for (int Idx = LastOpIdx; Idx >= 0; --Idx) {
-      Value *TrailingZeros = Builder.CreateCountTrailingZeroElems(
-          Builder.getInt64Ty(), State.get(getOperand(Idx)), true, Name);
+      Value *TrailingZeros =
+          State.VF.isScalar()
+              ? Builder.CreateZExt(
+                    Builder.CreateICmpEQ(State.get(getOperand(Idx)),
+                                         Builder.getFalse()),
+                    Builder.getInt64Ty())
+              : Builder.CreateCountTrailingZeroElems(Builder.getInt64Ty(),
+                                                     State.get(getOperand(Idx)),
+                                                     true, Name);
       Value *Current = Builder.CreateAdd(
           Builder.CreateMul(RuntimeVF, Builder.getInt64(Idx)), TrailingZeros);
       if (Res) {
@@ -1029,6 +1036,12 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
   switch (getOpcode()) {
   case Instruction::ExtractElement:
   case VPInstruction::ExtractLane: {
+    if (VF.isScalar()) {
+      // ExtractLane with VF=1 takes care of handling extracting across multiple
+      // parts.
+      return 0;
+    }
+
     // Add on the cost of extracting the element.
     auto *VecTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
     return Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,
@@ -1040,8 +1053,13 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
         Instruction::Or, cast<VectorType>(VecTy), std::nullopt, Ctx.CostKind);
   }
   case VPInstruction::FirstActiveLane: {
+    Type *ScalarTy = Ctx.Types.inferScalarType(getOperand(0));
+    if (VF.isScalar())
+      return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ScalarTy,
+                                        CmpInst::makeCmpResultType(ScalarTy),
+                                        CmpInst::ICMP_EQ, Ctx.CostKind);
     // Calculate the cost of determining the lane index.
-    auto *PredTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
+    auto *PredTy = toVectorTy(ScalarTy, VF);
     IntrinsicCostAttributes Attrs(Intrinsic::experimental_cttz_elts,
                                   Type::getInt64Ty(Ctx.LLVMCtx),
                                   {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e0bf241c73fd..56175e7f1814 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2983,9 +2983,11 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
     R->eraseFromParent();
 }
 
-void VPlanTransforms::handleUncountableEarlyExit(
-    VPBasicBlock *EarlyExitingVPBB, VPBasicBlock *EarlyExitVPBB, VPlan &Plan,
-    VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VFRange &Range) {
+void VPlanTransforms::handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
+                                                 VPBasicBlock *EarlyExitVPBB,
+                                                 VPlan &Plan,
+                                                 VPBasicBlock *HeaderVPBB,
+                                                 VPBasicBlock *LatchVPBB) {
   VPBlockBase *MiddleVPBB = LatchVPBB->getSuccessors()[0];
   if (!EarlyExitVPBB->getSinglePredecessor() &&
       EarlyExitVPBB->getPredecessors()[1] == MiddleVPBB) {
@@ -3038,13 +3040,7 @@ void VPlanTransforms::handleUncountableEarlyExit(
     }
 
     VPValue *IncomingFromEarlyExit = ExitIRI->getOperand(EarlyExitIdx);
-    auto IsVector = [](ElementCount VF) { return VF.isVector(); };
-    // When the VFs are vectors, need to add `extract` to get the incoming value
-    // from early exit. When the range contains scalar VF, limit the range to
-    // scalar VF to prevent mis-compilation for the range containing both scalar
-    // and vector VFs.
-    if (!IncomingFromEarlyExit->isLiveIn() &&
-        LoopVectorizationPlanner::getDecisionAndClampRange(IsVector, Range)) {
+    if (!IncomingFromEarlyExit->isLiveIn()) {
       // Update the incoming value from the early exit.
       VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
           VPInstruction::FirstActiveLane, {CondToEarlyExit}, nullptr,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 700b94621d5f..0b7769642214 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -70,8 +70,8 @@ struct VPlanTransforms {
               PredicatedScalarEvolution &PSE);
 
   /// Update \p Plan to account for all early exits.
-  LLVM_ABI_FOR_TEST static void
-  handleEarlyExits(VPlan &Plan, bool HasUncountableExit, VFRange &Range);
+  LLVM_ABI_FOR_TEST static void handleEarlyExits(VPlan &Plan,
+                                                 bool HasUncountableExit);
 
   /// If a check is needed to guard executing the scalar epilogue loop, it will
   /// be added to the middle block.
@@ -207,8 +207,7 @@ struct VPlanTransforms {
   static void handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
                                          VPBasicBlock *EarlyExitVPBB,
                                          VPlan &Plan, VPBasicBlock *HeaderVPBB,
-                                         VPBasicBlock *LatchVPBB,
-                                         VFRange &Range);
+                                         VPBasicBlock *LatchVPBB);
 
   /// Replace loop regions with explicit CFG.
   static void dissolveLoopRegions(VPlan &Plan);


Regression tests:
file: llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-only.ll
commands: opt -p loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -S %s
tests: test_name: iv_used_in_exit_with_math
test_body: define i8 @iv_used_in_exit_with_math(i8 noundef %g) {
entry:
  br label %loop.header

loop.header:                                      ; preds = %loop.latch, %entry
  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %s = shl nuw i8 1, %iv
  %a = and i8 %s, %g
  %c = icmp eq i8 %a, 0
  br i1 %c, label %loop.latch, label %return

loop.latch:                                       ; preds = %loop.header
  %iv.next = add nuw nsw i8 %iv, 1
  %ec = icmp eq i8 %iv.next, 4
  br i1 %ec, label %return, label %loop.header

return:                                           ; preds = %loop.latch, %loop.header
  %res.iv1 = phi i8 [ 32, %loop.latch ], [ %iv, %loop.header ]
  %res.iv2 = phi i8 [ 0, %loop.latch ], [ %iv, %loop.header ]
  %res = add i8 %res.iv1, %res.iv2
  ret i8 %res
}

additional_args: -src-unroll=32 -tgt-unroll=32