Bug type: crash
Component: LoopVectorize
Bug location: llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp: VPWidenLoadEVLRecipe::execute
VPWidenLoadEVLRecipe::print
VPWidenStoreEVLRecipe::execute
VPWidenStoreEVLRecipe::print
Issue title: [LV][VPlan] Crash due to disagreements on the VPlan cost v.s. legacy cost model
Labels: vectorizers
crash
Reported by: mshockwave

Given this input:
``` llvm
target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
target triple = "riscv64-unknown-linux-gnu"

define void @lshift_significand(i32 %n, ptr %0) {
entry:
  br label %for.cond

for.cond:                                         ; preds = %for.cond, %entry
  %i.0 = phi i32 [ 0, %entry ], [ 2, %for.cond ]
  %add = or i32 %n, %i.0
  %cmp1 = icmp eq i32 %add, 0
  br i1 %cmp1, label %for.cond, label %for.cond7

for.cond7:                                        ; preds = %for.body9, %for.cond
  %i.1 = phi i32 [ %i.0, %for.cond ], [ %inc15, %for.body9 ]
  %cmp8 = icmp ult i32 %i.1, 3
  br i1 %cmp8, label %for.body9, label %for.end16

for.body9:                                        ; preds = %for.cond7
  %sub11 = sub nuw i32 1, %i.1
  %idxprom12 = zext i32 %sub11 to i64
  %arrayidx13 = getelementptr [3 x i64], ptr %0, i64 0, i64 %idxprom12
  store i64 0, ptr %arrayidx13, align 8
  %inc15 = add i32 %i.1, 1
  br label %for.cond7

for.end16:                                        ; preds = %for.cond7
  ret void
}
```
And this command:
``` bash
opt -mcpu=sifive-p470 -O3 -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-tail-folding-style=data-with-evl input.ll -disable-output
```
We'll get the following assertion:
```
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp:7386: VectorizationFactor llvm::LoopVectorizationPlanner::computeBestVF(): Assertion `(BestFactor.Width == LegacyVF.Width || planContainsAdditionalSimplifications(getPlanFor(BestFactor.Width), CostCtx, OrigLoop)) && " VPlan cost model and legacy cost model disagreed"' failed.
```


Preliminary investigation shows that this is caused by the disagreement on (reverse) widen store's cost. First, this is the trace for VPlan's cost model:
```
LV: Scalar loop costs: 4.
Cost of 1 for VF vscale x 1: induction instruction   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
Cost of 0 for VF vscale x 1: induction instruction   %indvars.iv = phi i64 [ %spec.select, %entry ], [ %indvars.iv.next, %for.body9 ]
Cost of 1 for VF vscale x 1: exit condition instruction   %exitcond.not = icmp eq i64 %indvars.iv.next, 3
Cost of 0 for VF vscale x 1: EMIT vp<%4> = CANONICAL-INDUCTION ir<0>, vp<%12>
Cost of 0 for VF vscale x 1: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<%5> = phi ir<0>, vp<%11>
Cost of 0 for VF vscale x 1: EMIT vp<%6> = EXPLICIT-VECTOR-LENGTH vp<%5>, vp<%3>
Cost of 0 for VF vscale x 1: vp<%7> = DERIVED-IV ir<%spec.select> + vp<%5> * ir<1>
Cost of 0 for VF vscale x 1: vp<%8> = SCALAR-STEPS vp<%7>, ir<1>
Cost of 1 for VF vscale x 1: CLONE ir<%1> = sub nuw nsw ir<1>, vp<%8>
Cost of 0 for VF vscale x 1: CLONE ir<%arrayidx13> = getelementptr ir<%0>, ir<0>, ir<%1>
Cost of 0 for VF vscale x 1: vp<%9> = vector-pointer (reverse) ir<%arrayidx13>
Cost of 8 for VF vscale x 1: WIDEN vp.store vp<%9>, ir<0>, vp<%6>
Cost of 0 for VF vscale x 1: SCALAR-CAST vp<%10> = zext vp<%6> to i64
Cost of 0 for VF vscale x 1: EMIT vp<%11> = add vp<%10>, vp<%5>
Cost of 0 for VF vscale x 1: EMIT vp<%12> = add vp<%4>, vp<%0>
Cost of 0 for VF vscale x 1: EMIT branch-on-count vp<%12>, vp<%1>
Cost of 0 for VF vscale x 1: vector loop backedge
Cost for VF vscale x 1: 11
Cost of 1 for VF vscale x 2: induction instruction   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
Cost of 0 for VF vscale x 2: induction instruction   %indvars.iv = phi i64 [ %spec.select, %entry ], [ %indvars.iv.next, %for.body9 ]
Cost of 1 for VF vscale x 2: exit condition instruction   %exitcond.not = icmp eq i64 %indvars.iv.next, 3
Cost of 0 for VF vscale x 2: EMIT vp<%4> = CANONICAL-INDUCTION ir<0>, vp<%12>
Cost of 0 for VF vscale x 2: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<%5> = phi ir<0>, vp<%11>
Cost of 0 for VF vscale x 2: EMIT vp<%6> = EXPLICIT-VECTOR-LENGTH vp<%5>, vp<%3>
Cost of 0 for VF vscale x 2: vp<%7> = DERIVED-IV ir<%spec.select> + vp<%5> * ir<1>
Cost of 0 for VF vscale x 2: vp<%8> = SCALAR-STEPS vp<%7>, ir<1>
Cost of 1 for VF vscale x 2: CLONE ir<%1> = sub nuw nsw ir<1>, vp<%8>
Cost of 0 for VF vscale x 2: CLONE ir<%arrayidx13> = getelementptr ir<%0>, ir<0>, ir<%1>
Cost of 0 for VF vscale x 2: vp<%9> = vector-pointer (reverse) ir<%arrayidx13>
Cost of 14 for VF vscale x 2: WIDEN vp.store vp<%9>, ir<0>, vp<%6>
Cost of 0 for VF vscale x 2: SCALAR-CAST vp<%10> = zext vp<%6> to i64
Cost of 0 for VF vscale x 2: EMIT vp<%11> = add vp<%10>, vp<%5>
Cost of 0 for VF vscale x 2: EMIT vp<%12> = add vp<%4>, vp<%0>
Cost of 0 for VF vscale x 2: EMIT branch-on-count vp<%12>, vp<%1>
Cost of 0 for VF vscale x 2: vector loop backedge
Cost for VF vscale x 2: 17
```
With VPlan's cost model, we will eventually choose scalar loop, because when VF=vscale x 1, the final cost is `ceil(11 / 2) = 6`; when VF=vscale x 2, the final cost is `ceil(17 / 4) = 5`. Both of them are larger than the scalar cost, 4.

While with the legacy cost model:
```
LV: Found an estimated cost of 0 for VF 1 For instruction:   %indvars.iv = phi i64 [ %spec.select, %entry ], [ %indvars.iv.next, %for.body9 ]
LV: Found an estimated cost of 1 for VF 1 For instruction:   %1 = sub nuw nsw i64 1, %indvars.iv
LV: Found an estimated cost of 0 for VF 1 For instruction:   %arrayidx13 = getelementptr [3 x i64], ptr %0, i64 0, i64 %1
LV: Found an estimated cost of 1 for VF 1 For instruction:   store i64 0, ptr %arrayidx13, align 8
LV: Found an estimated cost of 1 for VF 1 For instruction:   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
LV: Found an estimated cost of 1 for VF 1 For instruction:   %exitcond.not = icmp eq i64 %indvars.iv.next, 3
LV: Found an estimated cost of 0 for VF 1 For instruction:   br i1 %exitcond.not, label %for.end16, label %for.body9
LV: Scalar loop costs: 4.
LV: Found an estimated cost of 0 for VF vscale x 1 For instruction:   %indvars.iv = phi i64 [ %spec.select, %entry ], [ %indvars.iv.next, %for.body9 ]
LV: Found an estimated cost of 1 for VF vscale x 1 For instruction:   %1 = sub nuw nsw i64 1, %indvars.iv
LV: Found an estimated cost of 0 for VF vscale x 1 For instruction:   %arrayidx13 = getelementptr [3 x i64], ptr %0, i64 0, i64 %1
LV: Found an estimated cost of 7 for VF vscale x 1 For instruction:   store i64 0, ptr %arrayidx13, align 8
LV: Found an estimated cost of 1 for VF vscale x 1 For instruction:   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
LV: Found an estimated cost of 1 for VF vscale x 1 For instruction:   %exitcond.not = icmp eq i64 %indvars.iv.next, 3
LV: Found an estimated cost of 0 for VF vscale x 1 For instruction:   br i1 %exitcond.not, label %for.end16, label %for.body9
LV: Vector loop of width vscale x 1 costs: 5 (assuming a minimum vscale of 2).
LV: Found an estimated cost of 0 for VF vscale x 2 For instruction:   %indvars.iv = phi i64 [ %spec.select, %entry ], [ %indvars.iv.next, %for.body9 ]
LV: Found an estimated cost of 1 for VF vscale x 2 For instruction:   %1 = sub nuw nsw i64 1, %indvars.iv
LV: Found an estimated cost of 0 for VF vscale x 2 For instruction:   %arrayidx13 = getelementptr [3 x i64], ptr %0, i64 0, i64 %1
LV: Found an estimated cost of 13 for VF vscale x 2 For instruction:   store i64 0, ptr %arrayidx13, align 8
LV: Found an estimated cost of 1 for VF vscale x 2 For instruction:   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
LV: Found an estimated cost of 1 for VF vscale x 2 For instruction:   %exitcond.not = icmp eq i64 %indvars.iv.next, 3
LV: Found an estimated cost of 0 for VF vscale x 2 For instruction:   br i1 %exitcond.not, label %for.end16, label %for.body9
LV: Vector loop of width vscale x 2 costs: 4 (assuming a minimum vscale of 2).
LV: Selecting VF: vscale x 2.
```
We will eventually choose VF=vscale x 2.

The key difference is the cost of `store i64 0, ptr %arrayidx13, align 8` v.s. cost of `WIDEN vp.store vp<%9>, ir<0>, vp<%6>`, where the latter is larger than the former by 1.

In both cases (the store instruction in VPlan cost model v.s. legacy cost model) their costs are computed by the base cost of store + the cost of shuffle due to being reverse store. I used debugger to confirm that they used the same shuffle cost, which means that for some reason, VPlan's cost model yield a slightly higher cost for store than that in the legacy cost model.


Comments: author: ElvisWang123
body: It seems that  the legacy cost model using `getMaskedMemoryOpCost` to query the instruction cost. But in the VPlan-based cost model using `getMemoryOpCost`. 

In this case, the mask in the  `VPWidenStoreEVLRecipe` is false since it is using the EVL recipe.
The legacy cost model will not check the tail folding type so it will using `getMaskedMemoryOpCost` instead of `getMemoryOpCost` . And the `MaskedMemoryCost` is slightly smaller than non-mask version.
author: ElvisWang123
body: Open a PR to fix this issue. #109644 

We remove the shuffle cost when the value stored by reversed vector  is loop invariant.
author: mshockwave
body: CC @fhahn 

Fix patch:
commit a068b974b199b0e7350da2d9506adc7df3995ce3
Author: Elvis Wang <elvis.wang@sifive.com>
Date:   Thu Sep 26 07:10:25 2024 +0800

    [VPlan] Implement VPWidenLoad/StoreEVLRecipe::computeCost().  (#109644)
    
    Currently the EVL recipes transfer the tail masking to the EVL.
    But in the legacy cost model, the mask exist and will calculate the
    instruction cost of the mask.
    To fix the difference between the VPlan-based cost model and the legacy
    cost model, we always calculate the instruction cost for the mask in the
    EVL recipes.
    
    Note that we should remove the mask cost in the EVL recipes when we
    don't need to compare to the legacy cost model.
    
    This patch also fixes #109468.

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index bbcfaf9e19cd..23a24ce293ef 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2709,6 +2709,10 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
   /// Generate the wide load or gather.
   void execute(VPTransformState &State) override;
 
+  /// Return the cost of this VPWidenLoadEVLRecipe.
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override;
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
@@ -2787,6 +2791,10 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
   /// Generate the wide store or scatter.
   void execute(VPTransformState &State) override;
 
+  /// Return the cost of this VPWidenStoreEVLRecipe.
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override;
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index dacba152611c..9a0aefb898e5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2267,6 +2267,31 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
   State.set(this, Res);
 }
 
+InstructionCost VPWidenLoadEVLRecipe::computeCost(ElementCount VF,
+                                                  VPCostContext &Ctx) const {
+  if (!Consecutive || IsMasked)
+    return VPWidenMemoryRecipe::computeCost(VF, Ctx);
+
+  // We need to use the getMaskedMemoryOpCost() instead of getMemoryOpCost()
+  // here because the EVL recipes using EVL to replace the tail mask. But in the
+  // legacy model, it will always calculate the cost of mask.
+  // TODO: Using getMemoryOpCost() instead of getMaskedMemoryOpCost when we
+  // don't need to compare to the legacy cost model.
+  Type *Ty = ToVectorTy(getLoadStoreType(&Ingredient), VF);
+  const Align Alignment =
+      getLoadStoreAlignment(const_cast<Instruction *>(&Ingredient));
+  unsigned AS =
+      getLoadStoreAddressSpace(const_cast<Instruction *>(&Ingredient));
+  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+  InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost(
+      Ingredient.getOpcode(), Ty, Alignment, AS, CostKind);
+  if (!Reverse)
+    return Cost;
+
+  return Cost + Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Reverse,
+                                       cast<VectorType>(Ty), {}, CostKind, 0);
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPWidenLoadEVLRecipe::print(raw_ostream &O, const Twine &Indent,
                                  VPSlotTracker &SlotTracker) const {
@@ -2363,6 +2388,31 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
   State.addMetadata(NewSI, SI);
 }
 
+InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF,
+                                                   VPCostContext &Ctx) const {
+  if (!Consecutive || IsMasked)
+    return VPWidenMemoryRecipe::computeCost(VF, Ctx);
+
+  // We need to use the getMaskedMemoryOpCost() instead of getMemoryOpCost()
+  // here because the EVL recipes using EVL to replace the tail mask. But in the
+  // legacy model, it will always calculate the cost of mask.
+  // TODO: Using getMemoryOpCost() instead of getMaskedMemoryOpCost when we
+  // don't need to compare to the legacy cost model.
+  Type *Ty = ToVectorTy(getLoadStoreType(&Ingredient), VF);
+  const Align Alignment =
+      getLoadStoreAlignment(const_cast<Instruction *>(&Ingredient));
+  unsigned AS =
+      getLoadStoreAddressSpace(const_cast<Instruction *>(&Ingredient));
+  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+  InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost(
+      Ingredient.getOpcode(), Ty, Alignment, AS, CostKind);
+  if (!Reverse)
+    return Cost;
+
+  return Cost + Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Reverse,
+                                       cast<VectorType>(Ty), {}, CostKind, 0);
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPWidenStoreEVLRecipe::print(raw_ostream &O, const Twine &Indent,
                                   VPSlotTracker &SlotTracker) const {


Regression tests:
file: llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
commands: opt < %s --prefer-predicate-over-epilogue=predicate-dont-vectorize --passes=loop-vectorize -mcpu=sifive-p470 -mattr=+v,+f -force-tail-folding-style=data-with-evl -S
tests: test_name: lshift_significand
test_body: target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
target triple = "riscv64-unknown-linux-gnu"

define void @lshift_significand(i32 %n, ptr nocapture writeonly %dst) {
entry:
  %cmp1.peel = icmp eq i32 %n, 0
  %spec.select = select i1 %cmp1.peel, i64 2, i64 0
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ %spec.select, %entry ], [ %iv.next, %loop ]
  %0 = sub nuw nsw i64 1, %iv
  %arrayidx13 = getelementptr i64, ptr %dst, i64 %0
  store i64 0, ptr %arrayidx13, align 8
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 3
  br i1 %exitcond.not, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}