Bug type: miscompilation
Component: LoopVectorize
Bug location: llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp: VPlanTransforms::adjustFixedOrderRecurrences
VPlanTransforms::clearReductionWrapFlags
collectUsersRecursively
createScalarIVSteps
legalizeAndOptimizeInductions
Issue title: [clang] Miscompilation at -O2/3
Labels: miscompilation
vectorizers
Reported by: cardigan1008

This code prints 8 at `-O0/1` and prints 1 at `-O2/3`:

```c
int printf(const char *, ...);
static char a;
static char *b = &a;
static int c;
short d;
void e() {
  short f[8];
  char **g[] = {&b, &b};
  c = 0;
  for (; c < 8; c = 81 + 462704684 + *b - 462704765 + c + 1)
    f[c] = 0;
  d = f[5];
}
int main() {
  e();
  printf("%d\n", c);
}
```

Compiler Explorer: https://godbolt.org/z/MEKz8oncP

Bisected to https://github.com/llvm/llvm-project/commit/df4a615c988f3ae56f7e68a7df86acb60f16493a, which was committed by @fhahn 

Comments: author: dtcxzyw
body: Reduced testcase: https://alive2.llvm.org/ce/z/6Bg63G
```
; bin/opt -passes=loop-vectorize test.ll -S
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

define i32 @src() {
entry:
  %f.i = alloca [8 x i16], align 16
  %conv.i = sext i8 0 to i32
  %invariant.op.i = add nsw i32 %conv.i, 1
  br label %for.body.i

for.body.i:                                       ; preds = %for.body.i, %entry
  %0 = phi i32 [ 0, %entry ], [ %add2.reass.i, %for.body.i ]
  %idxprom.i = sext i32 %0 to i64
  %arrayidx.i = getelementptr [8 x i16], ptr %f.i, i64 0, i64 %idxprom.i
  store i16 0, ptr %arrayidx.i, align 2
  %add2.reass.i = add i32 %invariant.op.i, %0
  %cmp.i = icmp slt i32 %add2.reass.i, 8
  br i1 %cmp.i, label %for.body.i, label %e.exit

e.exit:                                           ; preds = %for.body.i
  %add2.reass.i.lcssa = phi i32 [ %add2.reass.i, %for.body.i ]
  ret i32 %add2.reass.i.lcssa
}
```
author: fhahn
body: Looking into it now, thanks
author: yaoxinliu
body: Thank God! clang19 works as expected.

Fix patch:
commit f9369cc602272796c15de1065a782f812e791df3
Author: Florian Hahn <flo@fhahn.com>
Date:   Mon Jan 6 22:40:41 2025 +0000

    [VPlan] Make sure last IV increment value is available if needed.
    
    Legalize extract-from-ends using uniform VPReplicateRecipe of wide
    inductions to use regular VPReplicateRecipe, so the correct end value
    is available.
    
    Fixes https://github.com/llvm/llvm-project/issues/121745.

diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 395287bde76f..3e3f5adf73a0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -562,21 +562,63 @@ createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
   return Builder.createScalarIVSteps(InductionOpcode, FPBinOp, BaseIV, Step);
 }
 
+static SmallVector<VPUser *> collectUsersRecursively(VPValue *V) {
+  SetVector<VPUser *> Users(V->user_begin(), V->user_end());
+  for (unsigned I = 0; I != Users.size(); ++I) {
+    VPRecipeBase *Cur = cast<VPRecipeBase>(Users[I]);
+    if (isa<VPHeaderPHIRecipe>(Cur))
+      continue;
+    for (VPValue *V : Cur->definedValues())
+      Users.insert(V->user_begin(), V->user_end());
+  }
+  return Users.takeVector();
+}
+
 /// Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd
 /// (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as
 /// VPWidenPointerInductionRecipe will generate vectors only. If some users
 /// require vectors while other require scalars, the scalar uses need to extract
 /// the scalars from the generated vectors (Note that this is different to how
-/// int/fp inductions are handled). Also optimize VPWidenIntOrFpInductionRecipe,
-/// if any of its users needs scalar values, by providing them scalar steps
-/// built on the canonical scalar IV and update the original IV's users. This is
-/// an optional optimization to reduce the needs of vector extracts.
+/// int/fp inductions are handled). Legalize extract-from-ends using uniform
+/// VPReplicateRecipe of wide inductions to use regular VPReplicateRecipe, so
+/// the correct end value is available. Also optimize
+/// VPWidenIntOrFpInductionRecipe, if any of its users needs scalar values, by
+/// providing them scalar steps built on the canonical scalar IV and update the
+/// original IV's users. This is an optional optimization to reduce the needs of
+/// vector extracts.
 static void legalizeAndOptimizeInductions(VPlan &Plan) {
+  using namespace llvm::VPlanPatternMatch;
   SmallVector<VPRecipeBase *> ToRemove;
   VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
   bool HasOnlyVectorVFs = !Plan.hasVF(ElementCount::getFixed(1));
   VPBuilder Builder(HeaderVPBB, HeaderVPBB->getFirstNonPhi());
   for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
+    auto *PhiR = dyn_cast<VPHeaderPHIRecipe>(&Phi);
+    if (!PhiR)
+      break;
+
+    // Check if any uniform VPReplicateRecipes using the phi recipe are used by
+    // ExtractFromEnd. Those must be replaced by a regular VPReplicateRecipe to
+    // ensure the final value is available.
+    // TODO: Remove once uniformity analysis is done on VPlan.
+    for (VPUser *U : collectUsersRecursively(PhiR)) {
+      auto *ExitIRI = dyn_cast<VPIRInstruction>(U);
+      VPValue *Op;
+      if (!ExitIRI || !match(ExitIRI->getOperand(0),
+                             m_VPInstruction<VPInstruction::ExtractFromEnd>(
+                                 m_VPValue(Op), m_VPValue())))
+        continue;
+      auto *RepR = dyn_cast<VPReplicateRecipe>(Op);
+      if (!RepR || !RepR->isUniform())
+        continue;
+      assert(!RepR->isPredicated() && "RepR must not be predicated");
+      Instruction *I = RepR->getUnderlyingInstr();
+      auto *Clone =
+          new VPReplicateRecipe(I, RepR->operands(), /*IsUniform*/ false);
+      Clone->insertAfter(RepR);
+      RepR->replaceAllUsesWith(Clone);
+    }
+
     // Replace wide pointer inductions which have only their scalars used by
     // PtrAdd(IndStart, ScalarIVSteps (0, Step)).
     if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(&Phi)) {
@@ -1086,18 +1128,6 @@ bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan,
   return true;
 }
 
-static SmallVector<VPUser *> collectUsersRecursively(VPValue *V) {
-  SetVector<VPUser *> Users(V->user_begin(), V->user_end());
-  for (unsigned I = 0; I != Users.size(); ++I) {
-    VPRecipeBase *Cur = cast<VPRecipeBase>(Users[I]);
-    if (isa<VPHeaderPHIRecipe>(Cur))
-      continue;
-    for (VPValue *V : Cur->definedValues())
-      Users.insert(V->user_begin(), V->user_end());
-  }
-  return Users.takeVector();
-}
-
 void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
   for (VPRecipeBase &R :
        Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {


Regression tests:
file: llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
commands: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s
opt -S -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 < %s
tests: test_name: test_iv_uniform_with_outside_use_scev_simplification
test_body: define i32 @test_iv_uniform_with_outside_use_scev_simplification(ptr %dst) {
entry:
  %step.1 = sext i8 0 to i32
  %step.2 = add nsw i32 %step.1, 1
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
  %gep.dst = getelementptr inbounds i16, ptr %dst, i32 %iv
  store i16 0, ptr %gep.dst, align 2
  %iv.next = add i32 %step.2, %iv
  %cmp.i = icmp slt i32 %iv.next, 8
  br i1 %cmp.i, label %loop, label %e.exit

e.exit:                                           ; preds = %loop
  %res = phi i32 [ %iv.next, %loop ]
  ret i32 %res
}

additional_args: -src-unroll=8 -tgt-unroll=8