Bug type: crash
Component: LoopVectorize
Bug location: llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp: VPlanTransforms::addMinimumVectorEpilogueIterationCheck
Issue title: Assertion failure in VPlanTransforms.cpp
Labels: vectorizers
crash-on-valid
Reported by: ilovepi

We're seeing an assertion failure trip when building Fuchsia w/ coverage instrumentation. 

Here is a reproducer 
[llvm-profdata-a2a90a.zip](https://github.com/user-attachments/files/22727832/llvm-profdata-a2a90a.zip)

I'm working on reducing it, and @gulfemsavrun has started a bisect.

The toolchain we're qualifying is from Oct 2 (commit b147019f8b11cd491f331bd707f764786792665e).

```console
clang++: /usr/local/google/home/paulkirth/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp:3917: static DenseMap<const SCEV *, Value *> llvm::VPlanTransforms::expandSCEVs(VPlan &, ScalarEvolution &): Assertion `none_of(*Entry, IsaPred<VPExpandSCEVRecipe>) && "VPExpandSCEVRecipes must be at the beginning of the entry block, " "after any VPIRInstructions"' failed.
PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace, preprocessed source, and associated run script.
Stack dump:
0.      Program arguments: /usr/local/google/home/paulkirth/llvm-project/build/bin/clang++ -cc1 -triple x86_64-unknown-windows-msvc19.33.0 -O2 -emit-obj -massembler-fatal-warnings -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name llvm-profdata.cc -mrelocation-model pic -pic-level 2 -mframe-pointer=non-leaf -relaxed-aliasing -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -fms-volatile -ffreestanding -D_HAS_STATIC_RTTI=0 -target-cpu x86-64-v2 -mllvm -x86-branches-within-32B-boundaries -tune-cpu generic -gno-column-info -gcodeview -debug-info-kind=constructor -fdebug-compilation-dir=. -object-file-name=kernel.efi_x64-coverage\\obj\\src\\lib\\llvm-profdata\\libllvm-profdata.llvm-profdata.cc.o -mllvm -crash-diagnostics-dir=clang-crashreports -ffunction-sections -fdata-sections -fcoverage-compilation-dir=. -sys-header-deps -D _LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS -D _LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1 -D TOOLCHAIN_VERSION=mSdw4dJP68zDVUDZeMkK7B6_Qa4ra24Ty4wlu7xGETgC -D __Fuchsia_API_level__=4293918720 -D ZX_ASSERT_LEVEL=2 -D __PE_COFF__ -D WITH_FRAME_POINTERS=1 -D HAVE_LLVM_PROFDATA=1 -U _MSC_VER -U _MSC_FULL_VER -Wall -Wextra -Wconversion -Wextra-semi -Wimplicit-fallthrough -Wnewline-eof -Wstrict-prototypes -Wwrite-strings -Wno-sign-conversion -Wno-unused-parameter -Wnonportable-system-include-path -Wno-missing-field-initializers -Wno-extra-qualification -Wno-cast-function-type-mismatch -Wno-unknown-warning-option -Wno-missing-template-arg-list-after-template-kw -Wno-deprecated-pragma -Wno-nontrivial-memaccess -Wthread-safety -Wno-unknown-warning-option -Wno-thread-safety-reference-return -Werror -Wno-builtin-macro-redefined -Wformat=2 -Wmissing-declarations -Wvla -Wshadow -Wno-deprecated-this-capture -std=c++20 -fdeprecated-macro -fstack-size-section -ferror-limit 19 -ftemplate-backtrace-limit=0 -fvisibility=hidden -fvisibility-inlines-hidden -ftrivial-auto-var-init=pattern -fno-rtti -fno-use-cxa-atexit -fms-extensions -fms-compatibility -fms-compatibility-version=19.33 -fno-threadsafe-statics -fno-implicit-modules -fskip-odr-check-in-gmf -faligned-allocation -fnew-alignment=8 -fcolor-diagnostics -vectorize-loops -vectorize-slp -faddrsig -x c++ llvm-profdata-a2a90a.cpp
1.      <eof> parser at end of file
2.      Optimizer
3.      Running pass "function<eager-inv>(drop-unnecessary-assumes,float2int,lower-constant-intrinsics,loop(loop-rotate<header-duplication;no-prepare-for-lto>,loop-deletion),loop-distribute,inject-tli-mappings,loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only;>,infer-alignment,loop-load-elim,instcombine<max-iterations=1;no-verify-fixpoint>,simplifycfg<bonus-inst-threshold=1;forward-switch-cond;switch-range-to-icmp;switch-to-lookup;no-keep-loops;hoist-common-insts;no-hoist-loads-stores-with-cond-faulting;sink-common-insts;speculate-blocks;simplify-cond-branch;no-speculate-unpredictables>,slp-vectorizer,vector-combine,instcombine<max-iterations=1;no-verify-fixpoint>,loop-unroll<O2>,transform-warning,sroa<preserve-cfg>,infer-alignment,instcombine<max-iterations=1;no-verify-fixpoint>,loop-mssa(licm<allowspeculation>),alignment-from-assumptions,loop-sink,instsimplify,div-rem-pairs,tailcallelim,simplifycfg<bonus-inst-threshold=1;no-forward-switch-cond;switch-range-to-icmp;no-switch-to-lookup;keep-loops;no-hoist-common-insts;hoist-loads-stores-with-cond-faulting;no-sink-common-insts;speculate-blocks;simplify-cond-branch;speculate-unpredictables>)" on module "llvm-profdata-a2a90a.cpp"
4.      Running pass "loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only;>" on function "?MergeLiveData@LlvmProfdata@@QEAAXULiveData@1@@Z"
#0 0x00005634a0128b28 llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) /usr/local/google/home/paulkirth/llvm-project/llvm/lib/Support/Unix/Signals.inc:834:13
llvm-profdata-a2a90a.sh: line 4: 3184705 Aborted                 (core dumped) "$HOME/llvm-project/build/bin/clang++" "-cc1" "-triple" "x86_64-unknown-windows-msvc19.33.0" "-O2" "-emit-obj" "-massembler-fatal-warnings" "-disable-free" "-clear-ast-before-backend" "-disable-llvm-verifier" "-discard-value-names" "-main-file-name" "llvm-profdata.cc" "-mrelocation-model" "pic" "-pic-level" "2" "-mframe-pointer=non-leaf" "-relaxed-aliasing" "-fmath-errno" "-ffp-contract=on" "-fno-rounding-math" "-mconstructor-aliases" "-fms-volatile" "-ffreestanding" "-D_HAS_STATIC_RTTI=0" "-target-cpu" "x86-64-v2" "-mllvm" "-x86-branches-within-32B-boundaries" "-tune-cpu" "generic" "-gno-column-info" "-gcodeview" "-debug-info-kind=constructor" "-fdebug-compilation-dir=." "-object-file-name=kernel.efi_x64-coverage\\obj\\src\\lib\\llvm-profdata\\libllvm-profdata.llvm-profdata.cc.o" "-mllvm" "-crash-diagnostics-dir=clang-crashreports" "-ffunction-sections" "-fdata-sections" "-fcoverage-compilation-dir=." "-sys-header-deps" "-D" "_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS" "-D" "_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS=1" "-D" "TOOLCHAIN_VERSION=mSdw4dJP68zDVUDZeMkK7B6_Qa4ra24Ty4wlu7xGETgC" "-D" "__Fuchsia_API_level__=4293918720" "-D" "ZX_ASSERT_LEVEL=2" "-D" "__PE_COFF__" "-D" "WITH_FRAME_POINTERS=1" "-D" "HAVE_LLVM_PROFDATA=1" "-U" "_MSC_VER" "-U" "_MSC_FULL_VER" "-Wall" "-Wextra" "-Wconversion" "-Wextra-semi" "-Wimplicit-fallthrough" "-Wnewline-eof" "-Wstrict-prototypes" "-Wwrite-strings" "-Wno-sign-conversion" "-Wno-unused-parameter" "-Wnonportable-system-include-path" "-Wno-missing-field-initializers" "-Wno-extra-qualification" "-Wno-cast-function-type-mismatch" "-Wno-unknown-warning-option" "-Wno-missing-template-arg-list-after-template-kw" "-Wno-deprecated-pragma" "-Wno-nontrivial-memaccess" "-Wthread-safety" "-Wno-unknown-warning-option" "-Wno-thread-safety-reference-return" "-Werror" "-Wno-builtin-macro-redefined" "-Wformat=2" "-Wmissing-declarations" "-Wvla" "-Wshadow" "-Wno-deprecated-this-capture" "-std=c++20" "-fdeprecated-macro" "-fstack-size-section" "-ferror-limit" "19" "-ftemplate-backtrace-limit=0" "-fvisibility=hidden" "-fvisibility-inlines-hidden" "-ftrivial-auto-var-init=pattern" "-fno-rtti" "-fno-use-cxa-atexit" "-fms-extensions" "-fms-compatibility" "-fms-compatibility-version=19.33" "-fno-threadsafe-statics" "-fno-implicit-modules" "-fskip-odr-check-in-gmf" "-faligned-allocation" "-fnew-alignment=8" "-fcolor-diagnostics" "-vectorize-loops" "-vectorize-slp" "-faddrsig" "-x" "c++" "llvm-profdata-a2a90a.cpp"
```

Comments: author: ilovepi
body: bad.ll
```llvm
; *** IR Dump Before LoopVectorizePass on ?MergeLiveData@LlvmProfdata@@QEAAXULiveData@1@@Z ***
; ModuleID = 'reduced.ll'
source_filename = "reduced.ll"
target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-windows-msvc19.33.0"

@"?CountersBegin_placeholder@?A0xE34CF3CB@@3PADA" = external global [1 x i8]

; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write, inaccessiblemem: none)
define noalias noundef ptr @"?ProfCountersData@?A0xE34CF3CB@@YA?AV?$span@D$0?0@__ktl@std@@XZ"() local_unnamed_addr #0 {
  store i64 sub (i64 0, i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @"?CountersBegin_placeholder@?A0xE34CF3CB@@3PADA", i64 1) to i64)), ptr inttoptr (i64 8 to ptr), align 8
  ret ptr null
}

; Function Attrs: nofree norecurse nosync nounwind memory(write, inaccessiblemem: none)
define void @"?MergeLiveData@LlvmProfdata@@QEAAXULiveData@1@@Z"() local_unnamed_addr #1 {
  store i64 sub (i64 0, i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @"?CountersBegin_placeholder@?A0xE34CF3CB@@3PADA", i64 1) to i64)), ptr inttoptr (i64 8 to ptr), align 8
  %.not.i = icmp eq i64 sub (i64 0, i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @"?CountersBegin_placeholder@?A0xE34CF3CB@@3PADA", i64 1) to i64)), 0
  br i1 %.not.i, label %"??$MergeData@EUlogical_and@__ktl@std@@@?A0xE34CF3CB@@YAXV?$span@W4byte@std@@$0?0@__ktl@std@@V?$span@$$CBW4byte@std@@$0?0@12@@Z.exit", label %.lr.ph.i.preheader

.lr.ph.i.preheader:                               ; preds = %0
  br label %.lr.ph.i

.lr.ph.i:                                         ; preds = %.lr.ph.i.preheader, %.lr.ph.i
  %.01.i = phi i64 [ %2, %.lr.ph.i ], [ 0, %.lr.ph.i.preheader ]
  %1 = getelementptr i8, ptr null, i64 %.01.i
  store i8 poison, ptr %1, align 1
  %2 = add nuw i64 %.01.i, 1
  %exitcond.not.i = icmp eq i64 %2, sub (i64 0, i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @"?CountersBegin_placeholder@?A0xE34CF3CB@@3PADA", i64 1) to i64))
  br i1 %exitcond.not.i, label %"??$MergeData@EUlogical_and@__ktl@std@@@?A0xE34CF3CB@@YAXV?$span@W4byte@std@@$0?0@__ktl@std@@V?$span@$$CBW4byte@std@@$0?0@12@@Z.exit.loopexit", label %.lr.ph.i

"??$MergeData@EUlogical_and@__ktl@std@@@?A0xE34CF3CB@@YAXV?$span@W4byte@std@@$0?0@__ktl@std@@V?$span@$$CBW4byte@std@@$0?0@12@@Z.exit.loopexit": ; preds = %.lr.ph.i
  br label %"??$MergeData@EUlogical_and@__ktl@std@@@?A0xE34CF3CB@@YAXV?$span@W4byte@std@@$0?0@__ktl@std@@V?$span@$$CBW4byte@std@@$0?0@12@@Z.exit"

"??$MergeData@EUlogical_and@__ktl@std@@@?A0xE34CF3CB@@YAXV?$span@W4byte@std@@$0?0@__ktl@std@@V?$span@$$CBW4byte@std@@$0?0@12@@Z.exit": ; preds = %"??$MergeData@EUlogical_and@__ktl@std@@@?A0xE34CF3CB@@YAXV?$span@W4byte@std@@$0?0@__ktl@std@@V?$span@$$CBW4byte@std@@$0?0@12@@Z.exit.loopexit", %0
  ret void
}

; Function Attrs: nofree norecurse nosync nounwind memory(readwrite, inaccessiblemem: none)
define void @"?MergeCounters@?A0xE34CF3CB@@YAXV?$span@W4byte@std@@$0?0@__ktl@std@@V?$span@$$CBW4byte@std@@$0?0@23@@Z"() local_unnamed_addr #2 {
  tail call void @"??$MergeData@EUlogical_and@__ktl@std@@@?A0xE34CF3CB@@YAXV?$span@W4byte@std@@$0?0@__ktl@std@@V?$span@$$CBW4byte@std@@$0?0@12@@Z"()
  ret void
}

; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
define noalias noundef ptr @"??$?0PEAD@?$span@D$0?0@__ktl@std@@QEAA@PEAD_K@Z"(ptr writeonly captures(none) initializes((8, 16)) %0, i64 %1) local_unnamed_addr #3 {
  %3 = getelementptr i8, ptr %0, i64 8
  store i64 %1, ptr %3, align 8
  ret ptr null
}

; Function Attrs: nofree norecurse nosync nounwind memory(readwrite, inaccessiblemem: none)
define void @"??$MergeData@EUlogical_and@__ktl@std@@@?A0xE34CF3CB@@YAXV?$span@W4byte@std@@$0?0@__ktl@std@@V?$span@$$CBW4byte@std@@$0?0@12@@Z"() local_unnamed_addr #4 {
  %1 = load i64, ptr inttoptr (i64 8 to ptr), align 8
  %.not = icmp eq i64 %1, 0
  br i1 %.not, label %._crit_edge, label %.lr.ph

._crit_edge:                                      ; preds = %.lr.ph, %0
  ret void

.lr.ph:                                           ; preds = %0, %.lr.ph
  %.01 = phi i64 [ %3, %.lr.ph ], [ 0, %0 ]
  %2 = getelementptr i8, ptr null, i64 %.01
  store i8 poison, ptr %2, align 1
  %3 = add nuw i64 %.01, 1
  %exitcond.not = icmp eq i64 %3, %1
  br i1 %exitcond.not, label %._crit_edge, label %.lr.ph
}

; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(read, inaccessiblemem: none)
define i64 @"?size@?$span@E$0?0@__ktl@std@@QEBA_KXZ"() local_unnamed_addr #5 {
  %1 = load i64, ptr inttoptr (i64 8 to ptr), align 8
  ret i64 %1
}

; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
define noalias ptr @"??A?$span@E$0?0@__ktl@std@@QEBAAEAE_K@Z"(i64 %0) local_unnamed_addr #6 {
  %2 = getelementptr i8, ptr null, i64 %0
  ret ptr %2
}

attributes #0 = { mustprogress nofree norecurse nosync nounwind willreturn memory(write, inaccessiblemem: none) }
attributes #1 = { nofree norecurse nosync nounwind memory(write, inaccessiblemem: none) "no-builtins" }
attributes #2 = { nofree norecurse nosync nounwind memory(readwrite, inaccessiblemem: none) }
attributes #3 = { mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write) }
attributes #4 = { nofree norecurse nosync nounwind memory(readwrite, inaccessiblemem: none) "no-builtins" }
attributes #5 = { mustprogress nofree norecurse nosync nounwind willreturn memory(read, inaccessiblemem: none) }
attributes #6 = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }

```

`opt bad.ll -passes="loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only;>" -S`
author: ilovepi
body: got it down a bit more

```llvm
; RUN: opt %s -passes="loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only;>" -S
target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-windows-msvc19.33.0"

@"?CountersBegin_placeholder@?A0xE34CF3CB@@3PADA" = external global [1 x i8]

define void @"?MergeLiveData@LlvmProfdata@@QEAAXULiveData@1@@Z"() {
.lr.ph.i.preheader:
  br label %.lr.ph.i

.lr.ph.i:                                         ; preds = %.lr.ph.i, %.lr.ph.i.preheader
  %.01.i = phi i64 [ %1, %.lr.ph.i ], [ 0, %.lr.ph.i.preheader ]
  %0 = getelementptr i8, ptr null, i64 %.01.i
  store i8 0, ptr %0, align 1
  %1 = add i64 %.01.i, 1
  %exitcond.not.i = icmp eq i64 %.01.i, sub (i64 0, i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @"?CountersBegin_placeholder@?A0xE34CF3CB@@3PADA", i64 1) to i64))
  br i1 %exitcond.not.i, label %"??$MergeData@EUlogical_and@__ktl@std@@@?A0xE34CF3CB@@YAXV?$span@W4byte@std@@$0?0@__ktl@std@@V?$span@$$CBW4byte@std@@$0?0@12@@Z.exit.loopexit", label %.lr.ph.i

"??$MergeData@EUlogical_and@__ktl@std@@@?A0xE34CF3CB@@YAXV?$span@W4byte@std@@$0?0@__ktl@std@@V?$span@$$CBW4byte@std@@$0?0@12@@Z.exit.loopexit": ; preds = %.lr.ph.i
  ret void
}
```
author: ilovepi
body: ```cpp
// RUN: clang -cc1 -triple x86_64-unknown-windows-msvc19.33.0 -O2 -ffreestanding -std=c++20 -ftrivial-auto-var-init=pattern -fms-compatibility -vectorize-loops -emit-llvm %s
namespace std {
enum byte;
template <bool> using __enable_if_t = int;
template <bool __v> struct integral_constant {
  static const bool value = __v;
};
template <bool _Val> using _BoolConstant = integral_constant<_Val>;
template <class _Tp>
using __libcpp_remove_reference_t = __remove_reference_t(_Tp);
template <class _Tp>
using remove_reference_t = __libcpp_remove_reference_t<_Tp>;
template <class _Tp, class _Up>
using _IsSame = _BoolConstant<__is_same(_Tp, _Up)>;
template <class _Tp, class _Up>
concept __same_as_impl = _IsSame<_Tp, _Up>::value;
template <class _Tp, class _Up>
concept same_as = __same_as_impl<_Up, _Tp>;
template <bool, class _IfRes, class> using conditional_t = _IfRes;
template <class _Tp> using __remove_cvref_t = __remove_cvref(_Tp);
template <class _Tp> using remove_cvref_t = __remove_cvref_t<_Tp>;
template <class> class numeric_limits;
template <> struct numeric_limits<unsigned long long> {
  static constexpr long max() { return 0; }
};
constexpr size_t dynamic_extent = numeric_limits<size_t>::max();
template <typename, size_t = dynamic_extent> class span;
template <typename _Tp> struct span<_Tp> {
  using size_type = size_t;
  span subspan(size_type, size_type __count) { return {0, __count}; }
  size_type size() { return __size_; }
  size_type size_bytes() { return __size_; }
  _Tp &operator[](size_type __idx) { return __data_[__idx]; }
  _Tp *__data_;
  size_type __size_;
};
template <class, class _Rhs>
concept assignable_from = requires(_Rhs __rhs) { __rhs; };
template <class _Tp> constexpr bool is_object_v = assignable_from<_Tp, _Tp>;
template <template <class> class, class>
integral_constant<true> __sfinae_test_impl(int);
template <template <class> class _Templ, class... _Args>
using _IsValidExpansion = decltype(__sfinae_test_impl<_Templ, _Args...>(0));
template <class _Tp>
using __test_for_primary_template = __enable_if_t<_IsSame<_Tp, _Tp>::value>;
template <class _Tp>
using __is_primary_template =
    _IsValidExpansion<__test_for_primary_template, _Tp>;
template <class> struct __cond_value_type;
template <class _Tp>
  requires is_object_v<_Tp>
struct __cond_value_type<_Tp> {
  using value_type = _Tp;
};
template <class> struct indirectly_readable_traits;
template <class _Tp>
struct indirectly_readable_traits<_Tp *> : __cond_value_type<_Tp> {};
template <class _Tp>
concept __dereferenceable = requires(_Tp __t) { __t; };
template <__dereferenceable _Tp> using iter_reference_t = decltype(*_Tp());
template <class _Ip>
using iter_value_t =
    conditional_t<__is_primary_template<int>::value,
                  indirectly_readable_traits<_Ip>, int>::value_type;
template <class _Ip>
concept contiguous_iterator =
    same_as<iter_value_t<_Ip>, remove_cvref_t<iter_reference_t<_Ip>>>;
template <contiguous_iterator _It, class _EndOrSize>
span(_It, _EndOrSize) -> span<remove_reference_t<iter_reference_t<_It>>>;
} // namespace std
struct LiveData {
  std::span<std::byte> bitmap;
};
namespace std {
template <class> struct bit_or;
}
char BitmapBegin, BitmapBegin_0, BitmapEnd;
std::span<char> ProfBitmapData() {
  return std::span(&BitmapBegin_0, &BitmapEnd - &BitmapBegin);
}
template <typename T, template <typename> class>
void MergeData(std::span<std::byte> to) {
  std::span to_data{reinterpret_cast<T *>(0), to.size_bytes()};
  for (size_t i; i < to_data.size(); ++i)
    to_data[i] = 0;
}
template <typename T, template <typename> class Op, typename FromT>
void MergeSelfData(std::span<std::byte> to, std::span<FromT> from, char *) {
  MergeData<T, Op>(to.subspan(0, from.size_bytes()));
}
void MergeLiveData() {
  LiveData data;
  MergeSelfData<char, std::bit_or>(data.bitmap, ProfBitmapData(), "");
}
```
In case it helps, heres a reproducer from the frontend (e.g. incase there's a frontend change that caused bad IR to be emitted)
author: ilovepi
body: Also, this is still an issue at ToT  (8889377f5c867ddb07658f6286f4e58e64e9e30a from today).
author: ilovepi
body: Bisected to 2016af56525734a3ac80d7256434e49e561befc2, #157545, @fhahn can you take a look?
author: fhahn
body: Yep thanks for the heads up

Fix patch:
commit 70c1c8f8007746040e256186d1e23b65d7590e00
Author: Florian Hahn <flo@fhahn.com>
Date:   Tue Oct 7 21:15:58 2025 +0100

    [VPlan] Create SCEV expansion for epilogue check first.
    
    VPExpandSCEVRecipes must be at the beginning of the entry block.
    addMinimumEpilogueIterationCheck currently creates VPInstructions to
    compute the remaining iterations before potentially creating
    VPExpandSCEVRecipes.
    
    Fix this by first creating any SCEV expansions if needed.
    
    Fixes https://github.com/llvm/llvm-project/issues/162128.

diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index c8212af9f8e0..b36298fcad74 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -763,6 +763,8 @@ void VPlanTransforms::addMinimumVectorEpilogueIterationCheck(
   // Add the minimum iteration check for the epilogue vector loop.
   VPValue *TC = Plan.getOrAddLiveIn(TripCount);
   VPBuilder Builder(cast<VPBasicBlock>(Plan.getEntry()));
+  VPValue *VFxUF = Builder.createExpandSCEV(SE.getElementCount(
+      TripCount->getType(), (EpilogueVF * EpilogueUF), SCEV::FlagNUW));
   VPValue *Count = Builder.createNaryOp(
       Instruction::Sub, {TC, Plan.getOrAddLiveIn(VectorTripCount)},
       DebugLoc::getUnknown(), "n.vec.remaining");
@@ -770,9 +772,6 @@ void VPlanTransforms::addMinimumVectorEpilogueIterationCheck(
   // Generate code to check if the loop's trip count is less than VF * UF of
   // the vector epilogue loop.
   auto P = RequiresScalarEpilogue ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_ULT;
-  VPValue *VFxUF = Builder.createExpandSCEV(SE.getElementCount(
-      TripCount->getType(), (EpilogueVF * EpilogueUF), SCEV::FlagNUW));
-
   auto *CheckMinIters = Builder.createICmp(
       P, Count, VFxUF, DebugLoc::getUnknown(), "min.epilog.iters.check");
   VPInstruction *Branch =


Regression tests:
file: llvm/test/Transforms/LoopVectorize/epilog-vectorization-scev-expansion.ll
commands: opt -p loop-vectorize -force-vector-width=4 -enable-epilogue-vectorization -epilogue-vectorization-force-VF=4 -S %s
tests: test_name: <module>
test_body: 
@end = external global [128 x i8]

; Test case for https://github.com/llvm/llvm-project/issues/162128.
define void @test_epilogue_step_scev_expansion(ptr %dst) {
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %gep.dst = getelementptr i8, ptr %dst, i64 %iv
  store i8 0, ptr %gep.dst, align 1
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv, sub (i64 0, i64 ptrtoint (ptr getelementptr inbounds nuw (i8, ptr @end, i64 1) to i64))
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}