Bug type: crash
Component: LoopVectorize
Bug location: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp: LoopVectorizationPlanner::precomputeCosts
VPCostContext::getPredBlockCostDivisor
VPCostContext::useEmulatedMaskMemRefHack
llvm/lib/Transforms/Vectorize/VPlan.cpp: VPCostContext::getScalarizationOverhead
llvm/lib/Transforms/Vectorize/VPlanHelpers.h: VPCostContext
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp: VPReplicateRecipe::computeCost
Issue title: [LV]VPlan cost model and legacy cost model disagreed
Labels: vectorizers
crash-on-valid
Reported by: alexey-bataev

https://godbolt.org/z/TzKsnE8Kr
@fhahn looks like another one


Fix patch:
commit b3dcf485d23a574e4c377739a32468240a87c2fc
Author: Florian Hahn <flo@fhahn.com>
Date:   Fri Feb 13 21:16:52 2026 +0000

    [VPlan] Compute NumPredStores for VPReplicateRecipe costs in VPlan.
    
    Compute the number of predicated stores directly in VPlan instead of
    using CM.useEmulatedMaskMemRefHack(), which will only account for the
    number of predicated stores for the last VF the legacy cost model
    considered.
    
    Fixes https://github.com/llvm/llvm-project/issues/181183

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 499c5a31421e..b5feb3feea2e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -318,7 +318,7 @@ static cl::opt<bool> EnableLoadStoreRuntimeInterleave(
         "Enable runtime interleaving until load/store ports are saturated"));
 
 /// The number of stores in a loop that are allowed to need predication.
-static cl::opt<unsigned> NumberOfStoresToPredicate(
+cl::opt<unsigned> NumberOfStoresToPredicate(
     "vectorize-num-stores-pred", cl::init(1), cl::Hidden,
     cl::desc("Max number of stores to be predicated behind an if."));
 
@@ -6890,10 +6890,6 @@ unsigned VPCostContext::getPredBlockCostDivisor(BasicBlock *BB) const {
   return CM.getPredBlockCostDivisor(CostKind, BB);
 }
 
-bool VPCostContext::useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF) {
-  return CM.useEmulatedMaskMemRefHack(I, VF);
-}
-
 InstructionCost
 LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
                                           VPCostContext &CostCtx) const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 9b22c399d7ac..c266842f107e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -67,6 +67,8 @@ const char LLVMLoopVectorizeFollowupEpilogue[] =
 
 extern cl::opt<unsigned> ForceTargetInstructionCost;
 
+extern cl::opt<unsigned> NumberOfStoresToPredicate;
+
 static cl::opt<bool> PrintVPlansInDotFormat(
     "vplan-print-in-dot-format", cl::Hidden,
     cl::desc("Use dot format instead of plain text when dumping VPlans"));
@@ -1803,3 +1805,31 @@ InstructionCost VPCostContext::getScalarizationOverhead(
   return ScalarizationCost +
          TTI.getOperandsScalarizationOverhead(Tys, CostKind, VIC);
 }
+
+bool VPCostContext::useEmulatedMaskMemRefHack(const VPReplicateRecipe *R,
+                                              ElementCount VF) {
+  const Instruction *UI = R->getUnderlyingInstr();
+  if (isa<LoadInst>(UI))
+    return true;
+  assert(isa<StoreInst>(UI) && "R must either be a load or store");
+
+  if (!NumPredStores) {
+    // Count the number of predicated stores in the VPlan, caching the result.
+    const VPlan &Plan = *R->getParent()->getPlan();
+    NumPredStores = 0;
+    for (const VPRegionBlock *VPRB :
+         VPBlockUtils::blocksOnly<const VPRegionBlock>(
+             vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
+      assert(VPRB->isReplicator() && "must only contain replicate regions");
+      for (const VPBasicBlock *VPBB :
+           VPBlockUtils::blocksOnly<const VPBasicBlock>(
+               vp_depth_first_shallow(VPRB->getEntry()))) {
+        *NumPredStores += count_if(*VPBB, [](const VPRecipeBase &R) {
+          auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
+          return RepR && isa<StoreInst>(RepR->getUnderlyingInstr());
+        });
+      }
+    }
+  }
+  return *NumPredStores > NumberOfStoresToPredicate;
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index bab7e25cbf40..113ca8c4d0f7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -346,6 +346,9 @@ struct VPCostContext {
   PredicatedScalarEvolution &PSE;
   const Loop *L;
 
+  /// Number of predicated stores in the VPlan, computed on demand.
+  std::optional<unsigned> NumPredStores;
+
   VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
                 const VPlan &Plan, LoopVectorizationCostModel &CM,
                 TargetTransformInfo::TargetCostKind CostKind,
@@ -385,9 +388,8 @@ struct VPCostContext {
       bool AlwaysIncludeReplicatingR = false);
 
   /// Returns true if an artificially high cost for emulated masked memrefs
-  /// should be used. Forwards to
-  /// LoopVectorizationCostModel::useEmulatedMaskMemRefHack.
-  bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF);
+  /// should be used.
+  bool useEmulatedMaskMemRefHack(const VPReplicateRecipe *R, ElementCount VF);
 };
 
 /// This class can be used to assign names to VPValues. For VPValues without
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 389331fd958d..c15b9fb23060 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3570,7 +3570,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
           VecI1Ty, APInt::getAllOnes(VF.getFixedValue()),
           /*Insert=*/false, /*Extract=*/true, Ctx.CostKind);
 
-      if (Ctx.useEmulatedMaskMemRefHack(UI, VF)) {
+      if (Ctx.useEmulatedMaskMemRefHack(this, VF)) {
         // Artificially setting to a high enough value to practically disable
         // vectorization with such operations.
         return 3000000;


Regression tests:
file: llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
commands: opt -p loop-vectorize -S %s
opt -p loop-vectorize  -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S %s
tests: test_name: predicated_store
test_body: target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
target triple = "arm64-apple-macosx14.0.0"

define void @predicated_store(ptr %A, ptr noalias %B, ptr noalias %C, ptr %D, ptr %E, double %divisor, i64 %loop.count) #0 {
entry:
  br label %loop.header

loop.header:                                      ; preds = %loop.latch, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %gep.B = getelementptr double, ptr %B, i64 %iv
  %first.val = load double, ptr %B, align 8
  store double 0.000000e+00, ptr %D, align 8
  %coeff = load double, ptr %C, align 8
  %neg = fneg double %first.val
  %norm = fdiv double %neg, %divisor
  %scaled.coeff = fmul double %coeff, %norm
  %input.val = load double, ptr %gep.B, align 8
  %scaled.input = fmul double %input.val, %divisor
  %product = fmul double %scaled.coeff, %scaled.input
  %final = fmul double %product, %divisor
  %is.pos = fcmp ogt double %final, 0.000000e+00
  br i1 %is.pos, label %then, label %loop.latch

then:                                             ; preds = %loop.header
  store double 0.000000e+00, ptr %A, align 8
  %gep.E = getelementptr i8, ptr %E, i64 %iv
  store double 0.000000e+00, ptr %gep.E, align 8
  br label %loop.latch

loop.latch:                                       ; preds = %then, %loop.header
  store double 0.000000e+00, ptr null, align 8
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv, %loop.count
  br i1 %ec, label %exit, label %loop.header

exit:                                             ; preds = %loop.latch
  ret void
}

attributes #0 = { "target-cpu"="neoverse-v2" }