Bug type: crash
Component: LoopVectorize
Bug location: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp: LoopVectorizationCostModel::computeMaxVF
Issue title: [LoopVectorizer]  Assertion `EPResumeVal && "must have a resume value for the canonical IV"' failed.
Labels: vectorizers
crash
Reported by: JonPsson1

clang -Wno-incompatible-pointer-types -O3 -march=z13 -S -c crash19.i -o a.out -w -mllvm -disable-licm-promotion -mllvm -epilogue-vectorization-force-VF=2 

[crash19.tar.gz](https://github.com/user-attachments/files/18383534/crash19.tar.gz)

#9 0x000002aa3ff5e440 preparePlanForEpilogueVectorLoop
#12 0x000002aa3ffa098a llvm::LoopVectorizePass::run

@bmahjour 
@fhahn 

Comments: author: fhahn
body: @JonPsson1 Was the test case fuzzer generated? Just curious, because it had a loop with unsigned max as iteration count.
author: JonPsson1
body: > [@JonPsson1](https://github.com/JonPsson1) Was the test case fuzzer generated? Just curious, because it had a loop with unsigned max as iteration count.

No, it was a CSmith test case.
author: fhahn
body: @JonPsson1 Could you check if it fixes the full reproducer? If not, it would be great if you could share the full reproducer?

Fix patch:
commit 1de3dc7d23dd6b856efad3a3a04f2396328726d7
Author: Florian Hahn <flo@fhahn.com>
Date:   Tue Jan 14 22:07:38 2025 +0000

    [LV] Bail out early if BTC+1 wraps.
    
    Currently we fail to detect the case where BTC + 1 wraps, i.e. the
    vector trip count is 0, In those cases, the minimum iteration count
    check will fail, and the vector code will never be executed.
    
    Explicitly check for this condition in computeMaxVF and avoid trying to
    vectorize alltogether.
    
    Note that a number of tests needed to be updated, because the vector
    loop would never be executed given the input IR.
    
    Fixes https://github.com/llvm/llvm-project/issues/122558.

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 744faef19243..fe2fb5e9faae 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4052,7 +4052,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
     return FixedScalableVFPair::getNone();
   }
 
-  unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
+  ScalarEvolution *SE = PSE.getSE();
+  unsigned TC = SE->getSmallConstantTripCount(TheLoop);
   unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
   LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
   if (TC != MaxTC)
@@ -4064,6 +4065,22 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
     return FixedScalableVFPair::getNone();
   }
 
+  // If BTC matches the widest induction type and is -1 then the trip count
+  // computation will wrap to 0 and the vector trip count will be 0. Do not try
+  // to vectorize.
+  const SCEV *BTC = SE->getBackedgeTakenCount(TheLoop);
+  if (!isa<SCEVCouldNotCompute>(BTC) &&
+      BTC->getType()->getScalarSizeInBits() >=
+          Legal->getWidestInductionType()->getScalarSizeInBits() &&
+      SE->isKnownPredicate(CmpInst::ICMP_EQ, BTC,
+                           SE->getMinusOne(BTC->getType()))) {
+    reportVectorizationFailure(
+        "Trip count computation wrapped",
+        "backedge-taken count is -1, loop trip count wrapped to 0",
+        "TripCountWrapped", ORE, TheLoop);
+    return FixedScalableVFPair::getNone();
+  }
+
   switch (ScalarEpilogueStatus) {
   case CM_ScalarEpilogueAllowed:
     return computeFeasibleMaxVF(MaxTC, UserVF, false);


Regression tests:
file: llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
commands: opt -mattr=+dotprod -passes=loop-vectorize -force-vector-interleave=1 -S < %s
tests: test_name: dotp
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @dotp(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

file: llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll
commands: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -mattr=+i8mm,+dotprod -S < %s
opt -passes=loop-vectorize -enable-epilogue-vectorization=false -mattr=+dotprod -S < %s
tests: test_name: dotp_z_s
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @dotp_z_s(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = sext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

test_name: dotp_s_z
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @dotp_s_z(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = sext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

file: llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll
commands: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -mattr=+neon,+dotprod -force-vector-interleave=1 -S < %s
opt -passes=loop-vectorize -enable-epilogue-vectorization=false -mattr=+neon,+dotprod -S < %s
opt -passes=loop-vectorize -enable-epilogue-vectorization=false -mattr=+neon,+dotprod -force-vector-interleave=1 -vectorizer-maximize-bandwidth -S < %s
tests: test_name: dotp_unrolled
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum3 = phi i32 [ 0, %entry ], [ %add.a3, %for.body ]
  %accum2 = phi i32 [ 0, %entry ], [ %add.a2, %for.body ]
  %accum1 = phi i32 [ 0, %entry ], [ %add.a1, %for.body ]
  %accum0 = phi i32 [ 0, %entry ], [ %add.a0, %for.body ]
  %gep.a0 = getelementptr inbounds i8, ptr %a, i64 %iv
  %gep.b0 = getelementptr inbounds i8, ptr %b, i64 %iv
  %offset.1 = or disjoint i64 %iv, 1
  %gep.a1 = getelementptr inbounds i8, ptr %a, i64 %offset.1
  %gep.b1 = getelementptr inbounds i8, ptr %b, i64 %offset.1
  %offset.2 = or disjoint i64 %iv, 2
  %gep.a2 = getelementptr inbounds i8, ptr %a, i64 %offset.2
  %gep.b2 = getelementptr inbounds i8, ptr %b, i64 %offset.2
  %offset.3 = or disjoint i64 %iv, 3
  %gep.a3 = getelementptr inbounds i8, ptr %a, i64 %offset.3
  %gep.b3 = getelementptr inbounds i8, ptr %b, i64 %offset.3
  %load.a0 = load i8, ptr %gep.a0, align 1
  %ext.a0 = sext i8 %load.a0 to i32
  %load.b0 = load i8, ptr %gep.b0, align 1
  %ext.b0 = sext i8 %load.b0 to i32
  %mul.a0 = mul nsw i32 %ext.b0, %ext.a0
  %add.a0 = add nsw i32 %mul.a0, %accum0
  %load.a1 = load i8, ptr %gep.a1, align 1
  %ext.a1 = sext i8 %load.a1 to i32
  %load.b1 = load i8, ptr %gep.b1, align 1
  %ext.b1 = sext i8 %load.b1 to i32
  %mul.a1 = mul nsw i32 %ext.a1, %ext.b1
  %add.a1 = add nsw i32 %mul.a1, %accum1
  %load.a2 = load i8, ptr %gep.a2, align 1
  %ext.a2 = sext i8 %load.a2 to i32
  %load.b2 = load i8, ptr %gep.b2, align 1
  %ext.b2 = sext i8 %load.b2 to i32
  %mul.a2 = mul nsw i32 %ext.a2, %ext.b2
  %add.a2 = add nsw i32 %mul.a2, %accum2
  %load.a3 = load i8, ptr %gep.a3, align 1
  %ext.a3 = sext i8 %load.a3 to i32
  %load.b3 = load i8, ptr %gep.b3, align 1
  %ext.b3 = sext i8 %load.b3 to i32
  %mul.a3 = mul nsw i32 %ext.a3, %ext.b3
  %add.a3 = add nsw i32 %mul.a3, %accum3
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %num_in
  br i1 %exitcond.not, label %exit, label %for.body

exit:                                             ; preds = %for.body
  %result0 = add nsw i32 %add.a0, %add.a1
  %result1 = add nsw i32 %add.a2, %add.a3
  %result = add nsw i32 %result0, %result1
  ret i32 %result
}

test_name: dotp
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @dotp(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

test_name: not_dotp_predicated_pragma
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr inbounds i8, ptr %b, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = sext i8 %load.a to i32
  %gep.a2 = getelementptr inbounds i8, ptr %a, i64 %iv
  %load.b = load i8, ptr %gep.a2, align 1
  %ext.b = sext i8 %load.b to i32
  %mul = mul nsw i32 %ext.b, %ext.a
  %add = add nsw i32 %mul, %accum
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %N
  br i1 %exitcond.not, label %exit, label %for.body, !llvm.loop !0

exit:                                             ; preds = %for.body
  ret i32 %add
}

!0 = distinct !{!0, !1, !2, !3}
!1 = !{!"llvm.loop.mustprogress"}
!2 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
!3 = !{!"llvm.loop.vectorize.enable", i1 true}

test_name: not_dotp_not_loop_carried
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %mul, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

test_name: not_dotp_different_types
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp_different_types(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i16, ptr %gep.b, align 2
  %ext.b = zext i16 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

test_name: not_dotp_not_phi
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp_not_phi(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %ext.b
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

test_name: not_dotp_extend_user
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp_extend_user(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  %result = add i32 %add, %ext.b
  ret i32 %result
}

test_name: not_dotp_predicated
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp_predicated(i64 %N, ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = sext i8 %load.a to i32
  %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = sext i8 %load.b to i32
  %mul = mul nsw i32 %ext.b, %ext.a
  %add = add nsw i32 %mul, %accum
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %N
  br i1 %exitcond.not, label %exit, label %for.body

exit:                                             ; preds = %for.body
  ret i32 %add
}

file: llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
commands: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s
opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s
opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S < %s
tests: test_name: not_dotp_not_phi2
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define void @not_dotp_not_phi2(ptr %matrix, i32 %n) #0 {
entry:
  %cmp = icmp sgt i32 %n, 0
  br i1 %cmp, label %for.preheader, label %exit

for.preheader:                                    ; preds = %entry
  %load.a = load i8, ptr null, align 1
  %load.a1 = load i8, ptr inttoptr (i64 1 to ptr), align 1
  %a.ext = sext i8 %load.a to i32
  %a.ext1 = sext i8 %load.a1 to i32
  br label %for.body

for.body:                                         ; preds = %for.body, %for.preheader
  %iv = phi i32 [ %iv.next, %for.body ], [ 0, %for.preheader ]
  %ptr = phi ptr [ %scevgep, %for.body ], [ %matrix, %for.preheader ]
  %accum = phi i32 [ %add.1, %for.body ], [ 0, %for.preheader ]
  %gep.b = getelementptr i8, ptr %ptr, i64 1
  %gep.b1 = getelementptr i8, ptr %ptr, i64 2
  %load.b = load i8, ptr %gep.b, align 1
  %b.ext = sext i8 %load.b to i32
  %mul = mul nsw i32 %a.ext, %b.ext
  %add = add i32 %mul, %accum
  %load.b1 = load i8, ptr %gep.b1, align 1
  %b.ext1 = sext i8 %load.b1 to i32
  %mul.1 = mul nsw i32 %a.ext1, %b.ext1
  %add.1 = add i32 %mul.1, %add
  %scevgep = getelementptr i8, ptr %ptr, i64 16
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond.not = icmp eq i32 %iv.next, %n
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  %add.1.lcssa = phi i32 [ %add.1, %for.body ]
  %add.float = sitofp i32 %add.1.lcssa to float
  br label %exit

exit:                                             ; preds = %for.exit, %entry
  %result = phi float [ 0.000000e+00, %entry ], [ %add.float, %for.exit ]
  store float %result, ptr %matrix, align 4
  ret void
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

test_name: not_dotp_ext_outside_plan2
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i64 @not_dotp_ext_outside_plan2(ptr %a, i16 %b, i64 %n) #0 {
entry:
  %cmp = icmp eq i64 %n, 0
  br i1 %cmp, label %exit, label %for.ph

for.ph:                                           ; preds = %entry
  %ext.b = zext i16 %b to i64
  br label %for.body

for.body:                                         ; preds = %for.body, %for.ph
  %iv = phi i64 [ 0, %for.ph ], [ %iv.next, %for.body ]
  %accum = phi i64 [ 0, %for.ph ], [ %add, %for.body ]
  %gep.a = getelementptr inbounds nuw i16, ptr %a, i64 %iv
  %load.a = load i16, ptr %gep.a, align 2
  %ext.a = zext i16 %load.a to i64
  %mul = mul nuw nsw i64 %ext.b, %ext.a
  %add = add i64 %mul, %accum
  %iv.next = add nuw nsw i64 %iv, 1
  %cmp.1 = icmp eq i64 %iv.next, %n
  br i1 %cmp.1, label %exit, label %for.body

exit:                                             ; preds = %for.body, %entry
  %result = phi i64 [ 0, %entry ], [ %add, %for.body ]
  ret i64 %result
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

test_name: dotp_unrolled
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum3 = phi i32 [ 0, %entry ], [ %add.a3, %for.body ]
  %accum2 = phi i32 [ 0, %entry ], [ %add.a2, %for.body ]
  %accum1 = phi i32 [ 0, %entry ], [ %add.a1, %for.body ]
  %accum0 = phi i32 [ 0, %entry ], [ %add.a0, %for.body ]
  %gep.a0 = getelementptr inbounds i8, ptr %a, i64 %iv
  %gep.b0 = getelementptr inbounds i8, ptr %b, i64 %iv
  %offset.1 = or disjoint i64 %iv, 1
  %gep.a1 = getelementptr inbounds i8, ptr %a, i64 %offset.1
  %gep.b1 = getelementptr inbounds i8, ptr %b, i64 %offset.1
  %offset.2 = or disjoint i64 %iv, 2
  %gep.a2 = getelementptr inbounds i8, ptr %a, i64 %offset.2
  %gep.b2 = getelementptr inbounds i8, ptr %b, i64 %offset.2
  %offset.3 = or disjoint i64 %iv, 3
  %gep.a3 = getelementptr inbounds i8, ptr %a, i64 %offset.3
  %gep.b3 = getelementptr inbounds i8, ptr %b, i64 %offset.3
  %load.a0 = load i8, ptr %gep.a0, align 1
  %ext.a0 = sext i8 %load.a0 to i32
  %load.b0 = load i8, ptr %gep.b0, align 1
  %ext.b0 = sext i8 %load.b0 to i32
  %mul.a0 = mul nsw i32 %ext.b0, %ext.a0
  %add.a0 = add nsw i32 %mul.a0, %accum0
  %load.a1 = load i8, ptr %gep.a1, align 1
  %ext.a1 = sext i8 %load.a1 to i32
  %load.b1 = load i8, ptr %gep.b1, align 1
  %ext.b1 = sext i8 %load.b1 to i32
  %mul.a1 = mul nsw i32 %ext.a1, %ext.b1
  %add.a1 = add nsw i32 %mul.a1, %accum1
  %load.a2 = load i8, ptr %gep.a2, align 1
  %ext.a2 = sext i8 %load.a2 to i32
  %load.b2 = load i8, ptr %gep.b2, align 1
  %ext.b2 = sext i8 %load.b2 to i32
  %mul.a2 = mul nsw i32 %ext.a2, %ext.b2
  %add.a2 = add nsw i32 %mul.a2, %accum2
  %load.a3 = load i8, ptr %gep.a3, align 1
  %ext.a3 = sext i8 %load.a3 to i32
  %load.b3 = load i8, ptr %gep.b3, align 1
  %ext.b3 = sext i8 %load.b3 to i32
  %mul.a3 = mul nsw i32 %ext.a3, %ext.b3
  %add.a3 = add nsw i32 %mul.a3, %accum3
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %num_in
  br i1 %exitcond.not, label %exit, label %for.body

exit:                                             ; preds = %for.body
  %result0 = add nsw i32 %add.a0, %add.a1
  %result1 = add nsw i32 %add.a2, %add.a3
  %result = add nsw i32 %result0, %result1
  ret i32 %result
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

test_name: dotp
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @dotp(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

test_name: not_dotp_predicated_pragma
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @not_dotp_predicated_pragma(i64 %N, ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr inbounds i8, ptr %b, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = sext i8 %load.a to i32
  %gep.a2 = getelementptr inbounds i8, ptr %a, i64 %iv
  %load.b = load i8, ptr %gep.a2, align 1
  %ext.b = sext i8 %load.b to i32
  %mul = mul nsw i32 %ext.b, %ext.a
  %add = add nsw i32 %mul, %accum
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %N
  br i1 %exitcond.not, label %exit, label %for.body, !llvm.loop !0

exit:                                             ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

!0 = distinct !{!0, !1, !2, !3}
!1 = !{!"llvm.loop.mustprogress"}
!2 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
!3 = !{!"llvm.loop.vectorize.enable", i1 true}

test_name: dotp_cost_disagreement
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %i.iv = phi i64 [ 0, %entry ], [ %i.iv.next, %for.body ]
  %sum = phi i64 [ 0, %entry ], [ %add, %for.body ]
  %arrayidx = getelementptr inbounds nuw i8, ptr %a, i64 %i.iv
  %0 = load i8, ptr %arrayidx, align 1
  %conv = zext i8 %0 to i64
  %i.iv.next = add nuw nsw i64 %i.iv, 1
  %arrayidx2 = getelementptr inbounds nuw i8, ptr %b, i64 %i.iv.next
  %1 = load i8, ptr %arrayidx2, align 1
  %conv3 = zext i8 %1 to i64
  %mul = mul nuw nsw i64 %conv3, %conv
  %add = add i64 %sum, %mul
  %exitcond.not = icmp eq i64 %i.iv.next, 16
  br i1 %exitcond.not, label %exit, label %for.body

exit:                                             ; preds = %for.body
  ret i64 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

test_name: not_dotp_not_loop_carried
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %mul, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

test_name: not_dotp_different_types
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @not_dotp_different_types(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i16, ptr %gep.b, align 2
  %ext.b = zext i16 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

test_name: not_dotp_not_phi
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %ext.b
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

test_name: not_dotp_extend_user
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  %result = add i32 %add, %ext.b
  ret i32 %result
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

test_name: not_dotp_ext_outside_plan
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i64 @not_dotp_ext_outside_plan(ptr %a, i16 %b, i64 %n) #0 {
entry:
  %cmp = icmp eq i64 %n, 0
  br i1 %cmp, label %exit, label %for.ph

for.ph:                                           ; preds = %entry
  %ext.b = zext i16 %b to i64
  br label %for.body

for.body:                                         ; preds = %for.body, %for.ph
  %iv = phi i64 [ 0, %for.ph ], [ %iv.next, %for.body ]
  %accum = phi i64 [ 0, %for.ph ], [ %add, %for.body ]
  %gep.a = getelementptr inbounds nuw i16, ptr %a, i64 %iv
  %load.a = load i16, ptr %gep.a, align 2
  %ext.a = zext i16 %load.a to i64
  %mul = mul nuw nsw i64 %ext.a, %ext.b
  %add = add i64 %mul, %accum
  %iv.next = add nuw nsw i64 %iv, 1
  %cmp.1 = icmp eq i64 %iv.next, %n
  br i1 %cmp.1, label %exit, label %for.body

exit:                                             ; preds = %for.body, %entry
  %result = phi i64 [ 0, %entry ], [ %add, %for.body ]
  ret i64 %result
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

test_name: not_dotp_predicated
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

; Function Attrs: vscale_range(1,16)
define i32 @not_dotp_predicated(i64 %N, ptr %a, ptr %b) #0 {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = sext i8 %load.a to i32
  %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = sext i8 %load.b to i32
  %mul = mul nsw i32 %ext.b, %ext.a
  %add = add nsw i32 %mul, %accum
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %N
  br i1 %exitcond.not, label %exit, label %for.body

exit:                                             ; preds = %for.body
  ret i32 %add
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

file: llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-no-dotprod.ll
commands: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -mattr=+neon -S < %s
tests: test_name: not_dotp
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @not_dotp(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1000
  br i1 %exitcond.not, label %for.exit, label %for.body

for.exit:                                         ; preds = %for.body
  ret i32 %add
}

file: llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
commands: opt -p loop-vectorize -S %s
opt -p loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S %s
tests: test_name: trunc_store
test_body: target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
target triple = "arm64-apple-macosx14.0.0"

; Function Attrs: vscale_range(1,16)
define void @trunc_store(ptr %dst, ptr %src, i16 %x) #0 {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %x.ext = zext i16 %x to i64
  %l = load i64, ptr %src, align 8
  %and = and i64 %l, %x.ext
  %trunc = trunc i64 %and to i8
  %gep = getelementptr i8, ptr %dst, i64 %iv
  store i8 %trunc, ptr %gep, align 1
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1000
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

file: llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
commands: opt -mattr=+neon,+dotprod -passes=loop-vectorize -debug-only=loop-vectorize -force-vector-interleave=1 -disable-output %s 2>&1
tests: test_name: print_partial_reduction
test_body: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"

define i32 @print_partial_reduction(ptr %a, ptr %b) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.body, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
  %gep.a = getelementptr i8, ptr %a, i64 %iv
  %load.a = load i8, ptr %gep.a, align 1
  %ext.a = zext i8 %load.a to i32
  %gep.b = getelementptr i8, ptr %b, i64 %iv
  %load.b = load i8, ptr %gep.b, align 1
  %ext.b = zext i8 %load.b to i32
  %mul = mul i32 %ext.b, %ext.a
  %add = add i32 %mul, %accum
  %iv.next = add i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %exit, label %for.body

exit:                                             ; preds = %for.body
  ret i32 %add
}

file: llvm/test/Transforms/LoopVectorize/Hexagon/maximum-vf-crash.ll
commands: opt -mtriple=hexagon -hexagon-autohvx -passes=loop-vectorize -S < %s 2>&1
tests: test_name: f
test_body: target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048"

; Function Attrs: optsize
define i32 @f(ptr %src) #0 {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %g.016 = phi i32 [ 0, %entry ], [ %g.1.lcssa, %loop ]
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
  %0 = load i8, ptr %src, align 1
  %g.1.lcssa = add i32 %g.016, 1
  %iv.next = add nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, 1000
  br i1 %exitcond, label %exit, label %loop

exit:                                             ; preds = %loop
  ret i32 %g.1.lcssa
}

attributes #0 = { optsize "target-features"="+hvx-length128b" }

file: llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
commands: opt -p loop-vectorize -S %s
tests: test_name: second_lshr_operand_zero_via_scev
test_body: target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

define i64 @second_lshr_operand_zero_via_scev() {
entry:
  %ext.0 = sext i8 0 to i32
  br label %loops

loops:                                            ; preds = %loops, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loops ]
  %red = phi i64 [ 0, %entry ], [ %red.next, %loops ]
  %c = icmp eq i64 %iv, 0
  %and = and i64 %iv, 0
  %0 = trunc i64 %iv to i32
  %shr = lshr i32 %0, %ext.0
  %conv.1 = zext i32 %shr to i64
  %red.next.v = select i1 %c, i64 %and, i64 %conv.1
  %red.next = or i64 %red.next.v, %red
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1000
  br i1 %ec, label %exit, label %loops

exit:                                             ; preds = %loops
  %res = phi i64 [ %red.next, %loops ]
  ret i64 %res
}

file: llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
commands: opt < %s -passes=loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -S
tests: test_name: narrowed_reduction
test_body: target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.8.0"

define i32 @narrowed_reduction(ptr %a, i1 %cmp) #0 {
entry:
  %conv = zext i1 %cmp to i32
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i32 [ 1, %entry ], [ %inc, %loop ]
  %or13 = phi i32 [ 0, %entry ], [ %or, %loop ]
  %and = and i32 %or13, 1
  %or = or i32 %and, %conv
  %inc = add i32 %iv, 1
  %ec = icmp eq i32 %iv, 16
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret i32 %or
}

attributes #0 = { "target-cpu"="penryn" }

file: llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
commands: opt %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S
tests: test_name: drop_zext_nneg
test_body: target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-pc-linux-gnu"

; Function Attrs: noinline nounwind uwtable
define void @drop_zext_nneg(ptr noalias %p, ptr noalias %p1) #0 {
entry:
  br label %body

body:                                             ; preds = %else, %entry
  %iv = phi i64 [ %next, %else ], [ 0, %entry ]
  %0 = trunc i64 %iv to i32
  %c = icmp eq i32 %0, 0
  br i1 %c, label %then, label %else

then:                                             ; preds = %body
  %zext = zext nneg i32 %0 to i64
  %idx1 = getelementptr double, ptr %p, i64 %zext
  %idx2 = getelementptr double, ptr %p, i64 %zext
  %1 = load double, ptr %idx2, align 8
  br label %else

else:                                             ; preds = %then, %body
  %phi = phi double [ %1, %then ], [ 0.000000e+00, %body ]
  store double %phi, ptr %p1, align 8
  %next = add i64 %iv, 1
  %cmp = icmp eq i64 %next, 1024
  br i1 %cmp, label %exit, label %body

exit:                                             ; preds = %else
  ret void
}

attributes #0 = { noinline nounwind uwtable "target-features"="+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl" }

file: llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
commands: opt -p loop-vectorize -S %s
tests: test_name: test_free_instructions_feeding_geps_for_interleave_groups
test_body: target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.15.0"

define void @test_free_instructions_feeding_geps_for_interleave_groups(ptr noalias %p.invar, ptr noalias %dst.1, ptr noalias %dst.2) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %l.0 = load float, ptr %p.invar, align 4
  %iv.mul = shl i64 %iv, 2
  %gep.dst.19 = getelementptr float, ptr %dst.1, i64 %iv.mul
  store float %l.0, ptr %gep.dst.19, align 4
  %l.1 = load float, ptr %p.invar, align 4
  %add.1 = or disjoint i64 %iv.mul, 1
  %gep.dst.119 = getelementptr float, ptr %dst.1, i64 %add.1
  store float %l.1, ptr %gep.dst.119, align 4
  %add.2 = or disjoint i64 %iv.mul, 2
  %gep.dst.129 = getelementptr float, ptr %dst.1, i64 %add.2
  store float 0.000000e+00, ptr %gep.dst.129, align 4
  %add.3 = or disjoint i64 %iv.mul, 3
  %gep.dst.140 = getelementptr float, ptr %dst.1, i64 %add.3
  store float 0.000000e+00, ptr %gep.dst.140, align 4
  %l.2 = load float, ptr %p.invar, align 4
  %gep.dst.247 = getelementptr float, ptr %dst.2, i64 %iv.mul
  store float %l.2, ptr %gep.dst.247, align 4
  %gep.dst.255 = getelementptr float, ptr %dst.2, i64 %add.1
  store float 0.000000e+00, ptr %gep.dst.255, align 4
  %gep.dst.265 = getelementptr float, ptr %dst.2, i64 %add.2
  store float 0.000000e+00, ptr %gep.dst.265, align 4
  %gep.dst.276 = getelementptr float, ptr %dst.2, i64 %add.3
  store float 0.000000e+00, ptr %gep.dst.276, align 4
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

file: llvm/test/Transforms/LoopVectorize/X86/replicate-uniform-call.ll
commands: opt -p loop-vectorize -S %s
tests: test_name: smax_call_uniform
test_body: target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

define void @smax_call_uniform(ptr %dst, i64 %x) {
entry:
  %c = icmp ult i8 -68, -69
  %mul = mul nuw nsw i64 %x, 0
  br label %loop.header

loop.header:                                      ; preds = %loop.latch, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  br i1 %c, label %loop.latch, label %else

else:                                             ; preds = %loop.header
  %rem = urem i64 %mul, %x
  %smax = tail call i64 @llvm.smax.i64(i64 %rem, i64 0)
  br label %loop.latch

loop.latch:                                       ; preds = %else, %loop.header
  %p = phi i64 [ 1, %loop.header ], [ %smax, %else ]
  %add = add i64 %p, 1
  %gep = getelementptr i64, ptr %dst, i64 %add
  store i64 0, ptr %gep, align 8
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop.header

exit:                                             ; preds = %loop.latch
  ret void
}

; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
declare i64 @llvm.smax.i64(i64, i64) #0

attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }

file: llvm/test/Transforms/LoopVectorize/epilog-vectorization-vector-trip-count-zero.ll
commands: opt -p loop-vectorize -force-vector-width=2 -epilogue-vectorization-force-VF=2 -S %s
tests: test_name: vector_trip_count_0_as_btc_is_all_1
test_body: target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"

define void @vector_trip_count_0_as_btc_is_all_1(ptr %dst) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
  %iv.next = add i32 %iv, -1
  %gep = getelementptr inbounds i32, ptr %dst, i32 %iv
  store i32 %iv.next, ptr %gep, align 4
  %ec = icmp eq i32 %iv.next, 0
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

file: llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll
commands: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s
tests: test_name: test_resinking_required
test_body: define double @test_resinking_required(ptr %p, ptr noalias %a, ptr noalias %b) {
Entry:
  br label %Loop

Loop:                                             ; preds = %Loop, %Entry
  %for.1 = phi double [ %l1, %Loop ], [ 0.000000e+00, %Entry ]
  %for.2 = phi double [ %l2, %Loop ], [ 0.000000e+00, %Entry ]
  %for.3 = phi double [ %for.2, %Loop ], [ 0.000000e+00, %Entry ]
  %iv = phi i64 [ %iv.next, %Loop ], [ 0, %Entry ]
  %USE_2_FORS = fdiv double %for.3, %for.1
  %div = fdiv double 0.000000e+00, %for.1
  %l1 = load double, ptr %a, align 8
  %iv.next = add nuw nsw i64 %iv, 1
  %l2 = load double, ptr %b, align 8
  store double %div, ptr %p, align 8
  %cond = icmp eq i64 %iv.next, 1000
  br i1 %cond, label %End, label %Loop

End:                                              ; preds = %Loop
  %res.1 = fadd double %for.1, %for.2
  %res.2 = fadd double %res.1, %for.3
  ret double %res.2
}

file: llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
commands: opt -S -vectorize-num-stores-pred=1 -force-vector-width=1 -force-vector-interleave=2 -passes=loop-vectorize,simplifycfg -verify-loop-info -simplifycfg-require-and-preserve-domtree=1 < %s
opt -S -vectorize-num-stores-pred=1 -force-vector-width=1 -force-vector-interleave=2 -passes=loop-vectorize -verify-loop-info < %s
opt -S -vectorize-num-stores-pred=1 -force-vector-width=2 -force-vector-interleave=1 -passes=loop-vectorize,simplifycfg -verify-loop-info -simplifycfg-require-and-preserve-domtree=1 < %s
tests: test_name: minimal_bit_widths_with_aliasing_store
test_body: target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"

define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.inc, %entry
  %tmp0 = phi i64 [ %tmp6, %for.inc ], [ 0, %entry ]
  %tmp1 = phi i64 [ %tmp7, %for.inc ], [ 1000, %entry ]
  %tmp2 = getelementptr i8, ptr %ptr, i64 %tmp0
  %tmp3 = load i8, ptr %tmp2, align 1
  store i8 0, ptr %tmp2, align 1
  br i1 %c, label %if.then, label %for.inc

if.then:                                          ; preds = %for.body
  %tmp4 = zext i8 %tmp3 to i32
  %tmp5 = trunc i32 %tmp4 to i8
  store i8 %tmp5, ptr %tmp2, align 1
  br label %for.inc

for.inc:                                          ; preds = %if.then, %for.body
  %tmp6 = add nuw nsw i64 %tmp0, 1
  %tmp7 = add i64 %tmp1, -1
  %tmp8 = icmp eq i64 %tmp7, 0
  br i1 %tmp8, label %for.end, label %for.body

for.end:                                          ; preds = %for.inc
  ret void
}

file: llvm/test/Transforms/LoopVectorize/induction.ll
commands: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S
opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=1 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S
opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=2 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S
opt < %s -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S
opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=2 -force-vector-width=4 -force-widen-divrem-via-safe-divisor=0 -enable-interleaved-mem-accesses -S
tests: test_name: max_i32_backedgetaken
test_body: target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"

define i32 @max_i32_backedgetaken() {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %a.0 = phi i32 [ 1, %entry ], [ %a.0.and, %loop ]
  %b.0 = phi i32 [ 0, %entry ], [ %b.next, %loop ]
  %a.0.and = and i32 %a.0, 4
  %b.next = add i32 %b.0, -1
  %ec = icmp eq i32 %b.next, 0
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret i32 %a.0.and
}

file: llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
commands: opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -debug -disable-output %s 2>&1
opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -S %s
tests: test_name: scalarize_ptrtoint
test_body: define void @scalarize_ptrtoint(ptr %src, ptr %dst) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %gep = getelementptr ptr, ptr %src, i64 %iv
  %l = load ptr, ptr %gep, align 8
  %cast = ptrtoint ptr %l to i64
  %add = add i64 %cast, 10
  %cast.2 = inttoptr i64 %add to ptr
  store ptr %cast.2, ptr %dst, align 8
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

file: llvm/test/Transforms/LoopVectorize/is_fpclass.ll
commands: opt < %s -passes=loop-vectorize -force-vector-width=2 -S
tests: test_name: d
test_body: define void @d() {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %i = phi i64 [ 0, %entry ], [ %i7, %loop ]
  %i3 = load float, ptr null, align 4
  %i4 = getelementptr float, ptr @d, i64 %i
  %i5 = tail call i1 @llvm.is.fpclass.f32(float %i3, i32 0)
  %i6 = select i1 %i5, float 0.000000e+00, float 0.000000e+00
  store float %i6, ptr %i4, align 4
  %i7 = add i64 %i, 1
  %i8 = icmp eq i64 %i7, 128
  br i1 %i8, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
declare i1 @llvm.is.fpclass.f32(float, i32 immarg) #0

attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }

file: llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll
commands: opt -passes='loop-vectorize' -force-vector-width=4 -enable-epilogue-vectorization -epilogue-vectorization-force-VF=4 -S %s
opt -passes='loop-vectorize' -enable-epilogue-vectorization -epilogue-vectorization-force-VF=2 -S %s
tests: test_name: multiple_ivs_wide
test_body: target datalayout = "e-m:e-i64:64-n32:64-v128:128:128"

define void @multiple_ivs_wide(ptr %dst) {
entry:
  br label %loop

loop:                                             ; preds = %loop, %entry
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
  %iv.2 = phi i32 [ -64, %entry ], [ %iv.2.next, %loop ]
  %iv.next = add i32 %iv, 2
  %iv.2.next = add i32 %iv.2, 2
  %gep = getelementptr inbounds i32, ptr %dst, i32 %iv
  store i32 %iv.2.next, ptr %gep, align 4
  %ec = icmp eq i32 %iv.next, 128
  br i1 %ec, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

file: llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll
commands: opt < %s -passes='loop-vectorize' -force-vector-interleave=1 -S -force-vector-width=4 -debug-only=loop-accesses,loop-vectorize,loop-utils
tests: test_name: stride_check_known_via_loop_guard
test_body: target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"

define void @stride_check_known_via_loop_guard(ptr %C, ptr %A, i32 %Acols) {
entry:
  %pre.c = icmp ugt i32 %Acols, 0
  br i1 %pre.c, label %exit, label %outer.header

outer.header:                                     ; preds = %outer.latch, %entry
  %outer.iv = phi i32 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %mul.us = mul i32 %outer.iv, %Acols
  %arrayidx.us = getelementptr double, ptr %A, i32 %mul.us
  br label %inner

inner:                                            ; preds = %inner, %outer.header
  %inner.iv = phi i32 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
  %gep.C = getelementptr inbounds double, ptr %C, i32 %inner.iv
  %l = load double, ptr %arrayidx.us, align 8
  store double %l, ptr %gep.C, align 8
  %inner.iv.next = add i32 %inner.iv, 1
  %inner.c = icmp eq i32 %inner.iv.next, 1000
  br i1 %inner.c, label %outer.latch, label %inner

outer.latch:                                      ; preds = %inner
  %outer.iv.next = add i32 %outer.iv, 1
  %outer.c = icmp ult i32 %outer.iv, 128
  br i1 %outer.c, label %exit, label %outer.header

exit:                                             ; preds = %outer.latch, %entry
  ret void
}

file: llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll
commands: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -lv-strided-pointer-ivs=true -S %s
tests: test_name: step_direction_unknown
test_body: define void @step_direction_unknown(i32 %arg, ptr %dst) {
entry:
  %add = add i32 %arg, 1
  br label %loop

loop:                                             ; preds = %loop, %entry
  %phi = phi i64 [ 0, %entry ], [ %add2, %loop ]
  %trunc = trunc i64 %phi to i32
  %mul = mul i32 %add, %trunc
  %zext = zext i32 %mul to i64
  %getelementptr = getelementptr double, ptr %dst, i64 %zext
  store double 0.000000e+00, ptr %getelementptr, align 8
  %add2 = add i64 %phi, 1
  %icmp = icmp eq i64 %add2, 1024
  br i1 %icmp, label %exit, label %loop

exit:                                             ; preds = %loop
  ret void
}

test_name: integer_induction_wraps_scev_predicate_known
test_body: define void @integer_induction_wraps_scev_predicate_known(i32 %x, ptr %call, ptr %start) {
entry:
  %mul = shl i32 %x, 1
  br label %for.cond

for.cond:                                         ; preds = %for.cond, %entry
  %iv = phi i32 [ 30, %entry ], [ %inc, %for.cond ]
  %p.0 = phi ptr [ %start, %entry ], [ %add.ptr, %for.cond ]
  %add.ptr = getelementptr i8, ptr %p.0, i32 %mul
  %arrayidx = getelementptr ptr, ptr %call, i32 %iv
  store ptr %p.0, ptr %arrayidx, align 4
  %inc = add i32 %iv, 1
  %tobool.not = icmp eq i32 %iv, 1024
  br i1 %tobool.not, label %for.end, label %for.cond

for.end:                                          ; preds = %for.cond
  ret void
}

file: llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
commands: opt -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 -debug -disable-output %s 2>&1
tests: test_name: sink_with_sideeffects
test_body: target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"

define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
entry:
  br label %for.body

for.body:                                         ; preds = %for.inc, %entry
  %tmp0 = phi i64 [ %tmp6, %for.inc ], [ 0, %entry ]
  %tmp1 = phi i64 [ %tmp7, %for.inc ], [ 1024, %entry ]
  %tmp2 = getelementptr i8, ptr %ptr, i64 %tmp0
  %tmp3 = load i8, ptr %tmp2, align 1
  store i8 0, ptr %tmp2, align 1
  %tmp4 = zext i8 %tmp3 to i32
  %tmp5 = trunc i32 %tmp4 to i8
  br i1 %c, label %if.then, label %for.inc

if.then:                                          ; preds = %for.body
  store i8 %tmp5, ptr %tmp2, align 1
  br label %for.inc

for.inc:                                          ; preds = %if.then, %for.body
  %tmp6 = add nuw nsw i64 %tmp0, 1
  %tmp7 = add i64 %tmp1, -1
  %tmp8 = icmp eq i64 %tmp7, 0
  br i1 %tmp8, label %for.end, label %for.body

for.end:                                          ; preds = %for.inc
  ret void
}