Repository: llvm/llvm-project
Pull request #138298: [LV] Fix ScalarIVSteps vplan pattern matcher, remove m_CanonicalIV()
Author: login: huntergr-arm
Labels: name: vectorizers
name: llvm:transforms
Created: 2025-05-02T15:44:34Z | Merged: 2025-05-14T14:01:03Z

783a846 changed VPScalarIVStepsRecipe to take 3 arguments (adding
VF explicitly) instead of 2, but didn't change the corresponding
pattern matcher.

This matcher was only used in vputils::isHeaderMask, and no test
ever reached that function with a ScalarIVSteps recipe for the
value being matched -- it was always a WideCanonicalIV. So the
matcher bailed out immediately before checking arguments and
asserting that the number of arguments in the recipe was the
same provided by the matcher.

Since the constructors for ScalarIVSteps take 3 values, we should
be safe to update the matcher and guard it with a dedicated gtest.

m_CanonicalIV() on the other hand is removed; as a phi recipe it
may not have a consistent number of arguments to match, only
requiring one (the start value) when being constructed with the
assumption that a second incoming value is added for the backedge
later. In order to keep the matcher we would need to add multiple
matchers with different numbers of arguments for it depending on
what phase of vplan construction we were in, and ensure that we
never reorder matcher usage vs. vplan transformation. Since the
main IR PatternMatch.h doesn't contain any matchers for PHI nodes,
I think we can just remove it and match via m_Specific() using the
VPValue we get from Plan.getCanonicalIV().



Diff:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 58865c296ed8a..f2a7f16e19a79 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -461,21 +461,14 @@ m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
   return m_Select(Op0, m_True(), Op1);
 }
 
-using VPCanonicalIVPHI_match =
-    Recipe_match<std::tuple<>, 0, false, VPCanonicalIVPHIRecipe>;
-
-inline VPCanonicalIVPHI_match m_CanonicalIV() {
-  return VPCanonicalIVPHI_match();
-}
-
-template <typename Op0_t, typename Op1_t>
+template <typename Op0_t, typename Op1_t, typename Op2_t>
 using VPScalarIVSteps_match =
-    Recipe_match<std::tuple<Op0_t, Op1_t>, 0, false, VPScalarIVStepsRecipe>;
+    TernaryRecipe_match<Op0_t, Op1_t, Op2_t, 0, false, VPScalarIVStepsRecipe>;
 
-template <typename Op0_t, typename Op1_t>
-inline VPScalarIVSteps_match<Op0_t, Op1_t> m_ScalarIVSteps(const Op0_t &Op0,
-                                                           const Op1_t &Op1) {
-  return VPScalarIVSteps_match<Op0_t, Op1_t>(Op0, Op1);
+template <typename Op0_t, typename Op1_t, typename Op2_t>
+inline VPScalarIVSteps_match<Op0_t, Op1_t, Op2_t>
+m_ScalarIVSteps(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
+  return VPScalarIVSteps_match<Op0_t, Op1_t, Op2_t>({Op0, Op1, Op2});
 }
 
 template <typename Op0_t, typename Op1_t, typename Op2_t>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 2db4957409c8d..82b2ed242b0cb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -62,7 +62,9 @@ bool vputils::isHeaderMask(const VPValue *V, VPlan &Plan) {
 
   if (match(V, m_ActiveLaneMask(m_VPValue(A), m_VPValue(B))))
     return B == Plan.getTripCount() &&
-           (match(A, m_ScalarIVSteps(m_CanonicalIV(), m_SpecificInt(1))) ||
+           (match(A, m_ScalarIVSteps(m_Specific(Plan.getCanonicalIV()),
+                                     m_SpecificInt(1),
+                                     m_Specific(&Plan.getVF()))) ||
             IsWideCanonicalIV(A));
 
   return match(V, m_Binary<Instruction::ICmp>(m_VPValue(A), m_VPValue(B))) &&
diff --git a/llvm/unittests/Transforms/Vectorize/CMakeLists.txt b/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
index 0df39c41a9041..53eeff28c185f 100644
--- a/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
@@ -12,6 +12,7 @@ add_llvm_unittest(VectorizeTests
   VPlanTest.cpp
   VPDomTreeTest.cpp
   VPlanHCFGTest.cpp
+  VPlanPatternMatchTest.cpp
   VPlanSlpTest.cpp
   VPlanVerifierTest.cpp
   )
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp
new file mode 100644
index 0000000000000..e38b4fad80b0e
--- /dev/null
+++ b/llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp
@@ -0,0 +1,55 @@
+//===- llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp ------===//
+//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../lib/Transforms/Vectorize/VPlanPatternMatch.h"
+#include "../lib/Transforms/Vectorize/LoopVectorizationPlanner.h"
+#include "../lib/Transforms/Vectorize/VPlan.h"
+#include "../lib/Transforms/Vectorize/VPlanHelpers.h"
+#include "VPlanTestBase.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "gtest/gtest.h"
+
+namespace llvm {
+
+namespace {
+using VPPatternMatchTest = VPlanTestBase;
+
+TEST_F(VPPatternMatchTest, ScalarIVSteps) {
+  VPlan &Plan = getPlan();
+  VPBasicBlock *VPBB = Plan.createVPBasicBlock("");
+  VPBuilder Builder(VPBB);
+
+  IntegerType *I64Ty = IntegerType::get(C, 64);
+  VPValue *StartV = Plan.getOrAddLiveIn(ConstantInt::get(I64Ty, 0));
+  auto *CanonicalIVPHI = new VPCanonicalIVPHIRecipe(StartV, DebugLoc());
+  Builder.insert(CanonicalIVPHI);
+
+  VPValue *Inc = Plan.getOrAddLiveIn(ConstantInt::get(I64Ty, 1));
+  VPValue *VF = &Plan.getVF();
+  VPValue *Steps = Builder.createScalarIVSteps(
+      Instruction::Add, nullptr, CanonicalIVPHI, Inc, VF, DebugLoc());
+
+  VPValue *Inc2 = Plan.getOrAddLiveIn(ConstantInt::get(I64Ty, 2));
+  VPValue *Steps2 = Builder.createScalarIVSteps(
+      Instruction::Add, nullptr, CanonicalIVPHI, Inc2, VF, DebugLoc());
+
+  using namespace VPlanPatternMatch;
+
+  ASSERT_TRUE(match(Steps, m_ScalarIVSteps(m_Specific(CanonicalIVPHI),
+                                           m_SpecificInt(1), m_Specific(VF))));
+  ASSERT_FALSE(
+      match(Steps2, m_ScalarIVSteps(m_Specific(CanonicalIVPHI),
+                                    m_SpecificInt(1), m_Specific(VF))));
+  ASSERT_TRUE(match(Steps2, m_ScalarIVSteps(m_Specific(CanonicalIVPHI),
+                                            m_SpecificInt(2), m_Specific(VF))));
+}
+
+} // namespace
+} // namespace llvm


Reviews:
user: login: fhahn
body: Thanks for the fix!
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-05T20:09:25Z
user: login: huntergr-arm
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-06T11:00:53Z
user: login: david-arm
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-09T10:39:07Z
user: login: fhahn
body: LGTM, with the formatting fixed
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-13T19:52:24Z

Review comments:
diff_hunk: @@ -0,0 +1,46 @@
+//===- llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp ------===//
path: llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp
user: login: fhahn
body: From the description, I assume it is not possible to expose this currently with an IR unit test?
created_at: 2025-05-05T20:09:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,46 @@
+//===- llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp ------===//
path: llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp
user: login: huntergr-arm
body: Looks like `vputils::isHeaderMask` is only called from `collectAllHeaderMasks`, explicitly on users of WideCanonicalIVs. So I guess something changed at some point such that we never executed that path on recipes besides a wide canonical iv, and those matchers are now effectively unused.

`collectAllHeaderMasks` itself is only called from `VPlanTransforms::addActiveLaneMask`, which contains an assert that there is a `VPWidenCanonicalIVRecipe` among the users of the canonical IV.

The reason I encountered this is trying to use the matchers for my early-exit-with-store PR, and finding they didn't work.
created_at: 2025-05-06T11:00:53Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,46 @@
+//===- llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp ------===//
+//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../lib/Transforms/Vectorize/VPlanPatternMatch.h"
+#include "../lib/Transforms/Vectorize/LoopVectorizationPlanner.h"
+#include "../lib/Transforms/Vectorize/VPlan.h"
+#include "../lib/Transforms/Vectorize/VPlanHelpers.h"
+#include "VPlanTestBase.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "gtest/gtest.h"
+
+namespace llvm {
+
+namespace {
+using VPPatternMatchTest = VPlanTestBase;
+
+TEST_F(VPPatternMatchTest, ScalarIVSteps) {
+  VPlan &Plan = getPlan();
+  VPBasicBlock *VPBB = Plan.createVPBasicBlock("");
+  VPBuilder Builder(VPBB);
+
+  IntegerType *I64Ty = IntegerType::get(C, 64);
+  VPValue *StartV = Plan.getOrAddLiveIn(ConstantInt::get(I64Ty, 0));
+  auto *CanonicalIVPHI = new VPCanonicalIVPHIRecipe(StartV, DebugLoc());
+  Builder.insert(CanonicalIVPHI);
+
+  VPValue *Inc = Plan.getOrAddLiveIn(ConstantInt::get(I64Ty, 1));
+  VPValue *VF = &Plan.getVF();
+  VPValue *Steps = Builder.createScalarIVSteps(
+      Instruction::Add, nullptr, CanonicalIVPHI, Inc, VF, DebugLoc());
+
+  using namespace VPlanPatternMatch;
+
+  ASSERT_TRUE(match(Steps, m_ScalarIVSteps(m_Specific(CanonicalIVPHI),
+                                           m_SpecificInt(1), m_Specific(VF))));
path: llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp
user: login: david-arm
body: Is it worth having an ASSERT_FALSE for a scalar IV steps with a step != 1?
created_at: 2025-05-09T10:39:07Z
author_association: CONTRIBUTOR

PR comments:
user: login: llvmbot
created_at: 2025-05-02T15:45:09Z
body: 
<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-vectorizers

Author: Graham Hunter (huntergr-arm)

<details>
<summary>Changes</summary>

783a846 changed VPScalarIVStepsRecipe to take 3 arguments (adding
VF explicitly) instead of 2, but didn't change the corresponding
pattern matcher.

This matcher was only used in vputils::isHeaderMask, and no test
ever reached that function with a ScalarIVSteps recipe for the
value being matched -- it was always a WideCanonicalIV. So the
matcher bailed out immediately before checking arguments and
asserting that the number of arguments in the recipe was the
same provided by the matcher.

Since the constructors for ScalarIVSteps take 3 values, we should
be safe to update the matcher and guard it with a dedicated gtest.

m_CanonicalIV() on the other hand is removed; as a phi recipe it
may not have a consistent number of arguments to match, only
requiring one (the start value) when being constructed with the
assumption that a second incoming value is added for the backedge
later. In order to keep the matcher we would need to add multiple
matchers with different numbers of arguments for it depending on
what phase of vplan construction we were in, and ensure that we
never reorder matcher usage vs. vplan transformation. Since the
main IR PatternMatch.h doesn't contain any matchers for PHI nodes,
I think we can just remove it and match via m_Specific() using the
VPValue we get from Plan.getCanonicalIV().


---
Full diff: https://github.com/llvm/llvm-project/pull/138298.diff


4 Files Affected:

- (modified) llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h (+6-13) 
- (modified) llvm/lib/Transforms/Vectorize/VPlanUtils.cpp (+3-1) 
- (modified) llvm/unittests/Transforms/Vectorize/CMakeLists.txt (+1) 
- (added) llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp (+46) 


``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 58865c296ed8a..f2a7f16e19a79 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -461,21 +461,14 @@ m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
   return m_Select(Op0, m_True(), Op1);
 }
 
-using VPCanonicalIVPHI_match =
-    Recipe_match<std::tuple<>, 0, false, VPCanonicalIVPHIRecipe>;
-
-inline VPCanonicalIVPHI_match m_CanonicalIV() {
-  return VPCanonicalIVPHI_match();
-}
-
-template <typename Op0_t, typename Op1_t>
+template <typename Op0_t, typename Op1_t, typename Op2_t>
 using VPScalarIVSteps_match =
-    Recipe_match<std::tuple<Op0_t, Op1_t>, 0, false, VPScalarIVStepsRecipe>;
+    TernaryRecipe_match<Op0_t, Op1_t, Op2_t, 0, false, VPScalarIVStepsRecipe>;
 
-template <typename Op0_t, typename Op1_t>
-inline VPScalarIVSteps_match<Op0_t, Op1_t> m_ScalarIVSteps(const Op0_t &Op0,
-                                                           const Op1_t &Op1) {
-  return VPScalarIVSteps_match<Op0_t, Op1_t>(Op0, Op1);
+template <typename Op0_t, typename Op1_t, typename Op2_t>
+inline VPScalarIVSteps_match<Op0_t, Op1_t, Op2_t>
+m_ScalarIVSteps(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
+  return VPScalarIVSteps_match<Op0_t, Op1_t, Op2_t>({Op0, Op1, Op2});
 }
 
 template <typename Op0_t, typename Op1_t, typename Op2_t>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 2db4957409c8d..82b2ed242b0cb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -62,7 +62,9 @@ bool vputils::isHeaderMask(const VPValue *V, VPlan &Plan) {
 
   if (match(V, m_ActiveLaneMask(m_VPValue(A), m_VPValue(B))))
     return B == Plan.getTripCount() &&
-           (match(A, m_ScalarIVSteps(m_CanonicalIV(), m_SpecificInt(1))) ||
+           (match(A, m_ScalarIVSteps(m_Specific(Plan.getCanonicalIV()),
+                                     m_SpecificInt(1),
+                                     m_Specific(&Plan.getVF()))) ||
             IsWideCanonicalIV(A));
 
   return match(V, m_Binary<Instruction::ICmp>(m_VPValue(A), m_VPValue(B))) &&
diff --git a/llvm/unittests/Transforms/Vectorize/CMakeLists.txt b/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
index 0df39c41a9041..53eeff28c185f 100644
--- a/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
@@ -12,6 +12,7 @@ add_llvm_unittest(VectorizeTests
   VPlanTest.cpp
   VPDomTreeTest.cpp
   VPlanHCFGTest.cpp
+  VPlanPatternMatchTest.cpp
   VPlanSlpTest.cpp
   VPlanVerifierTest.cpp
   )
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp
new file mode 100644
index 0000000000000..e974af769ad27
--- /dev/null
+++ b/llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp
@@ -0,0 +1,46 @@
+//===- llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp ------===//
+//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../lib/Transforms/Vectorize/VPlanPatternMatch.h"
+#include "../lib/Transforms/Vectorize/LoopVectorizationPlanner.h"
+#include "../lib/Transforms/Vectorize/VPlan.h"
+#include "../lib/Transforms/Vectorize/VPlanHelpers.h"
+#include "VPlanTestBase.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "gtest/gtest.h"
+
+namespace llvm {
+
+namespace {
+using VPPatternMatchTest = VPlanTestBase;
+
+TEST_F(VPPatternMatchTest, ScalarIVSteps) {
+  VPlan &Plan = getPlan();
+  VPBasicBlock *VPBB = Plan.createVPBasicBlock("");
+  VPBuilder Builder(VPBB);
+
+  IntegerType *I64Ty = IntegerType::get(C, 64);
+  VPValue *StartV = Plan.getOrAddLiveIn(ConstantInt::get(I64Ty, 0));
+  auto *CanonicalIVPHI = new VPCanonicalIVPHIRecipe(StartV, DebugLoc());
+  Builder.insert(CanonicalIVPHI);
+
+  VPValue *Inc = Plan.getOrAddLiveIn(ConstantInt::get(I64Ty, 1));
+  VPValue *VF = &Plan.getVF();
+  VPValue *Steps = Builder.createScalarIVSteps(
+      Instruction::Add, nullptr, CanonicalIVPHI, Inc, VF, DebugLoc());
+
+  using namespace VPlanPatternMatch;
+
+  ASSERT_TRUE(match(Steps, m_ScalarIVSteps(m_Specific(CanonicalIVPHI),
+                                           m_SpecificInt(1), m_Specific(VF))));
+}
+
+} // namespace
+} // namespace llvm

``````````

</details>

author_association: MEMBER
user: login: llvmbot
created_at: 2025-05-02T15:45:10Z
body: 
<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-llvm-transforms

Author: Graham Hunter (huntergr-arm)

<details>
<summary>Changes</summary>

783a846 changed VPScalarIVStepsRecipe to take 3 arguments (adding
VF explicitly) instead of 2, but didn't change the corresponding
pattern matcher.

This matcher was only used in vputils::isHeaderMask, and no test
ever reached that function with a ScalarIVSteps recipe for the
value being matched -- it was always a WideCanonicalIV. So the
matcher bailed out immediately before checking arguments and
asserting that the number of arguments in the recipe was the
same provided by the matcher.

Since the constructors for ScalarIVSteps take 3 values, we should
be safe to update the matcher and guard it with a dedicated gtest.

m_CanonicalIV() on the other hand is removed; as a phi recipe it
may not have a consistent number of arguments to match, only
requiring one (the start value) when being constructed with the
assumption that a second incoming value is added for the backedge
later. In order to keep the matcher we would need to add multiple
matchers with different numbers of arguments for it depending on
what phase of vplan construction we were in, and ensure that we
never reorder matcher usage vs. vplan transformation. Since the
main IR PatternMatch.h doesn't contain any matchers for PHI nodes,
I think we can just remove it and match via m_Specific() using the
VPValue we get from Plan.getCanonicalIV().


---
Full diff: https://github.com/llvm/llvm-project/pull/138298.diff


4 Files Affected:

- (modified) llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h (+6-13) 
- (modified) llvm/lib/Transforms/Vectorize/VPlanUtils.cpp (+3-1) 
- (modified) llvm/unittests/Transforms/Vectorize/CMakeLists.txt (+1) 
- (added) llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp (+46) 


``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 58865c296ed8a..f2a7f16e19a79 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -461,21 +461,14 @@ m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
   return m_Select(Op0, m_True(), Op1);
 }
 
-using VPCanonicalIVPHI_match =
-    Recipe_match<std::tuple<>, 0, false, VPCanonicalIVPHIRecipe>;
-
-inline VPCanonicalIVPHI_match m_CanonicalIV() {
-  return VPCanonicalIVPHI_match();
-}
-
-template <typename Op0_t, typename Op1_t>
+template <typename Op0_t, typename Op1_t, typename Op2_t>
 using VPScalarIVSteps_match =
-    Recipe_match<std::tuple<Op0_t, Op1_t>, 0, false, VPScalarIVStepsRecipe>;
+    TernaryRecipe_match<Op0_t, Op1_t, Op2_t, 0, false, VPScalarIVStepsRecipe>;
 
-template <typename Op0_t, typename Op1_t>
-inline VPScalarIVSteps_match<Op0_t, Op1_t> m_ScalarIVSteps(const Op0_t &Op0,
-                                                           const Op1_t &Op1) {
-  return VPScalarIVSteps_match<Op0_t, Op1_t>(Op0, Op1);
+template <typename Op0_t, typename Op1_t, typename Op2_t>
+inline VPScalarIVSteps_match<Op0_t, Op1_t, Op2_t>
+m_ScalarIVSteps(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
+  return VPScalarIVSteps_match<Op0_t, Op1_t, Op2_t>({Op0, Op1, Op2});
 }
 
 template <typename Op0_t, typename Op1_t, typename Op2_t>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 2db4957409c8d..82b2ed242b0cb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -62,7 +62,9 @@ bool vputils::isHeaderMask(const VPValue *V, VPlan &Plan) {
 
   if (match(V, m_ActiveLaneMask(m_VPValue(A), m_VPValue(B))))
     return B == Plan.getTripCount() &&
-           (match(A, m_ScalarIVSteps(m_CanonicalIV(), m_SpecificInt(1))) ||
+           (match(A, m_ScalarIVSteps(m_Specific(Plan.getCanonicalIV()),
+                                     m_SpecificInt(1),
+                                     m_Specific(&Plan.getVF()))) ||
             IsWideCanonicalIV(A));
 
   return match(V, m_Binary<Instruction::ICmp>(m_VPValue(A), m_VPValue(B))) &&
diff --git a/llvm/unittests/Transforms/Vectorize/CMakeLists.txt b/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
index 0df39c41a9041..53eeff28c185f 100644
--- a/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/unittests/Transforms/Vectorize/CMakeLists.txt
@@ -12,6 +12,7 @@ add_llvm_unittest(VectorizeTests
   VPlanTest.cpp
   VPDomTreeTest.cpp
   VPlanHCFGTest.cpp
+  VPlanPatternMatchTest.cpp
   VPlanSlpTest.cpp
   VPlanVerifierTest.cpp
   )
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp
new file mode 100644
index 0000000000000..e974af769ad27
--- /dev/null
+++ b/llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp
@@ -0,0 +1,46 @@
+//===- llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp ------===//
+//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../lib/Transforms/Vectorize/VPlanPatternMatch.h"
+#include "../lib/Transforms/Vectorize/LoopVectorizationPlanner.h"
+#include "../lib/Transforms/Vectorize/VPlan.h"
+#include "../lib/Transforms/Vectorize/VPlanHelpers.h"
+#include "VPlanTestBase.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "gtest/gtest.h"
+
+namespace llvm {
+
+namespace {
+using VPPatternMatchTest = VPlanTestBase;
+
+TEST_F(VPPatternMatchTest, ScalarIVSteps) {
+  VPlan &Plan = getPlan();
+  VPBasicBlock *VPBB = Plan.createVPBasicBlock("");
+  VPBuilder Builder(VPBB);
+
+  IntegerType *I64Ty = IntegerType::get(C, 64);
+  VPValue *StartV = Plan.getOrAddLiveIn(ConstantInt::get(I64Ty, 0));
+  auto *CanonicalIVPHI = new VPCanonicalIVPHIRecipe(StartV, DebugLoc());
+  Builder.insert(CanonicalIVPHI);
+
+  VPValue *Inc = Plan.getOrAddLiveIn(ConstantInt::get(I64Ty, 1));
+  VPValue *VF = &Plan.getVF();
+  VPValue *Steps = Builder.createScalarIVSteps(
+      Instruction::Add, nullptr, CanonicalIVPHI, Inc, VF, DebugLoc());
+
+  using namespace VPlanPatternMatch;
+
+  ASSERT_TRUE(match(Steps, m_ScalarIVSteps(m_Specific(CanonicalIVPHI),
+                                           m_SpecificInt(1), m_Specific(VF))));
+}
+
+} // namespace
+} // namespace llvm

``````````

</details>

author_association: MEMBER
user: login: github-actions[bot]
created_at: 2025-05-13T15:53:04Z
body: <!--LLVM CODE FORMAT COMMENT: {clang-format}-->

:white_check_mark: With the latest revision this PR passed the C/C++ code formatter.
author_association: NONE
user: login: llvm-ci
created_at: 2025-05-14T18:35:46Z
body: LLVM Buildbot has detected a new failure on builder `llvm-clang-x86_64-gcc-ubuntu-no-asserts` running on `doug-worker-6` while building `llvm` at step 6 "test-build-unified-tree-check-all".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/202/builds/1236

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 6 (test-build-unified-tree-check-all) failure: test (failure)
******************** TEST 'AddressSanitizer-x86_64-linux-dynamic :: TestCases/asan_lsan_deadlock.cpp' FAILED ********************
Exit Code: 1

Command Output (stderr):
--
/home/buildbot/buildbot-root/gcc-no-asserts/build/./bin/clang  --driver-mode=g++ -fsanitize=address -mno-omit-leaf-frame-pointer -fno-omit-frame-pointer -fno-optimize-sibling-calls -gline-tables-only  -m64  -shared-libasan -O0 /home/buildbot/buildbot-root/gcc-no-asserts/llvm-project/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp -o /home/buildbot/buildbot-root/gcc-no-asserts/build/runtimes/runtimes-bins/compiler-rt/test/asan/X86_64LinuxDynamicConfig/TestCases/Output/asan_lsan_deadlock.cpp.tmp # RUN: at line 4
+ /home/buildbot/buildbot-root/gcc-no-asserts/build/./bin/clang --driver-mode=g++ -fsanitize=address -mno-omit-leaf-frame-pointer -fno-omit-frame-pointer -fno-optimize-sibling-calls -gline-tables-only -m64 -shared-libasan -O0 /home/buildbot/buildbot-root/gcc-no-asserts/llvm-project/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp -o /home/buildbot/buildbot-root/gcc-no-asserts/build/runtimes/runtimes-bins/compiler-rt/test/asan/X86_64LinuxDynamicConfig/TestCases/Output/asan_lsan_deadlock.cpp.tmp
env ASAN_OPTIONS=detect_leaks=1 not  /home/buildbot/buildbot-root/gcc-no-asserts/build/runtimes/runtimes-bins/compiler-rt/test/asan/X86_64LinuxDynamicConfig/TestCases/Output/asan_lsan_deadlock.cpp.tmp 2>&1 | FileCheck /home/buildbot/buildbot-root/gcc-no-asserts/llvm-project/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp # RUN: at line 5
+ env ASAN_OPTIONS=detect_leaks=1 not /home/buildbot/buildbot-root/gcc-no-asserts/build/runtimes/runtimes-bins/compiler-rt/test/asan/X86_64LinuxDynamicConfig/TestCases/Output/asan_lsan_deadlock.cpp.tmp
+ FileCheck /home/buildbot/buildbot-root/gcc-no-asserts/llvm-project/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp
[1m/home/buildbot/buildbot-root/gcc-no-asserts/llvm-project/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp:58:12: [0m[0;1;31merror: [0m[1mCHECK: expected string not found in input
[0m // CHECK: SUMMARY: AddressSanitizer: stack-buffer-overflow
[0;1;32m           ^
[0m[1m<stdin>:1:1: [0m[0;1;30mnote: [0m[1mscanning from here
[0m=================================================================
[0;1;32m^
[0m[1m<stdin>:2:10: [0m[0;1;30mnote: [0m[1mpossible intended match here
[0m==2007349==ERROR: AddressSanitizer: stack-buffer-overflow on address 0x78d0ba800034 at pc 0x5ecca7a22220 bp 0x78d0b89fdce0 sp 0x78d0b89fdcd8
[0;1;32m         ^
[0m
Input file: <stdin>
Check file: /home/buildbot/buildbot-root/gcc-no-asserts/llvm-project/compiler-rt/test/asan/TestCases/asan_lsan_deadlock.cpp

-dump-input=help explains the following input dump.

Input was:
<<<<<<
[1m[0m[0;1;30m            1: [0m[1m[0;1;46m================================================================= [0m
[0;1;31mcheck:58'0     X~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ error: no match found
[0m[0;1;30m            2: [0m[1m[0;1;46m==2007349==ERROR: AddressSanitizer: stack-buffer-overflow on address 0x78d0ba800034 at pc 0x5ecca7a22220 bp 0x78d0b89fdce0 sp 0x78d0b89fdcd8 [0m
[0;1;31mcheck:58'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[0m[0;1;35mcheck:58'1              ?                                                                                                                                    possible intended match
[0m[0;1;30m            3: [0m[1m[0;1;46mWRITE of size 4 at 0x78d0ba800034 thread T2 [0m
[0;1;31mcheck:58'0     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[0m>>>>>>

--

********************


```

</details>
author_association: NONE
user: login: llvm-ci
created_at: 2025-05-15T12:27:49Z
body: LLVM Buildbot has detected a new failure on builder `lld-x86_64-win` running on `as-worker-93` while building `llvm` at step 7 "test-build-unified-tree-check-all".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/146/builds/2921

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 7 (test-build-unified-tree-check-all) failure: test (failure)
******************** TEST 'LLVM-Unit :: Support/./SupportTests.exe/90/95' FAILED ********************
Script(shard):
--
GTEST_OUTPUT=json:C:\a\lld-x86_64-win\build\unittests\Support\.\SupportTests.exe-LLVM-Unit-8112-90-95.json GTEST_SHUFFLE=0 GTEST_TOTAL_SHARDS=95 GTEST_SHARD_INDEX=90 C:\a\lld-x86_64-win\build\unittests\Support\.\SupportTests.exe
--

Script:
--
C:\a\lld-x86_64-win\build\unittests\Support\.\SupportTests.exe --gtest_filter=ProgramEnvTest.CreateProcessLongPath
--
C:\a\lld-x86_64-win\llvm-project\llvm\unittests\Support\ProgramTest.cpp(160): error: Expected equality of these values:
  0
  RC
    Which is: -2

C:\a\lld-x86_64-win\llvm-project\llvm\unittests\Support\ProgramTest.cpp(163): error: fs::remove(Twine(LongPath)): did not return errc::success.
error number: 13
error message: permission denied



C:\a\lld-x86_64-win\llvm-project\llvm\unittests\Support\ProgramTest.cpp:160
Expected equality of these values:
  0
  RC
    Which is: -2

C:\a\lld-x86_64-win\llvm-project\llvm\unittests\Support\ProgramTest.cpp:163
fs::remove(Twine(LongPath)): did not return errc::success.
error number: 13
error message: permission denied




********************


```

</details>
author_association: NONE