Repository: llvm/llvm-project
File: llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp (C++)

Code before:
  //   data.phi = phi ir<default.val>, vp<new.data>
  //
  //   ...'data' and 'compare' created by existing nodes...
  //
  //   ...new recipes introduced to determine whether to update the reduction
  //      values or keep the current one.
  //   any.active = i1 any-of ir<compare>
  //   new.mask = select vp<any.active>, ir<compare>, vp<mask.phi>
  //   new.data = select vp<any.active>, ir<data>, ir<data.phi>
  //
  // middle.block:
  //   ...extract-last-active replaces compute-reduction-result.
  //   result = extract-last-active vp<new.data>, vp<new.mask>, ir<default.val>

  VPValue *HeaderMask = vputils::findHeaderMask(Plan);
  for (auto &Phi : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
    auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&Phi);
    if (!PhiR || !RecurrenceDescriptor::isFindLastRecurrenceKind(
                     PhiR->getRecurrenceKind()))
      continue;

    // Find the condition for the select/blend.
    VPValue *BackedgeSelect = PhiR->getBackedgeRecipe().getVPSingleValue();
    VPValue *CondSelect = BackedgeSelect;

    // If there's a header mask the backedge select will not be the find-last
    // select.
    if (HeaderMask && !match(BackedgeSelect,
                             m_Select(m_Specific(HeaderMask),
                                      m_VPValue(CondSelect), m_Specific(PhiR))))
      return false;

    VPValue *Cond = nullptr, *Op1 = nullptr, *Op2 = nullptr;

    // If we're matching a blend rather than a select, there should be one
    // incoming value which is the data, then all other incoming values should
    // be the phi.
    auto MatchBlend = [&](VPRecipeBase *R) {
      auto *Blend = dyn_cast<VPBlendRecipe>(R);
      if (!Blend)
        return false;
      assert(!Blend->isNormalized() && "must run before blend normalizaion");
      unsigned NumIncomingDataValues = 0;
      for (unsigned I = 0; I < Blend->getNumIncomingValues(); ++I) {
        VPValue *Incoming = Blend->getIncomingValue(I);
        if (Incoming != PhiR) {
          ++NumIncomingDataValues;
          Cond = Blend->getMask(I);
          Op1 = Incoming;
          Op2 = PhiR;
        }

Review comment (suggestion):
nit:
```suggestion
    // If there's a header mask, the backedge select will not be the find-last
```

Code after:
  //   data.phi = phi ir<default.val>, vp<new.data>
  //
  //   ...'data' and 'compare' created by existing nodes...
  //
  //   ...new recipes introduced to determine whether to update the reduction
  //      values or keep the current one.
  //   any.active = i1 any-of ir<compare>
  //   new.mask = select vp<any.active>, ir<compare>, vp<mask.phi>
  //   new.data = select vp<any.active>, ir<data>, ir<data.phi>
  //
  // middle.block:
  //   ...extract-last-active replaces compute-reduction-result.
  //   result = extract-last-active vp<new.data>, vp<new.mask>, ir<default.val>

  VPValue *HeaderMask = vputils::findHeaderMask(Plan);
  for (auto &Phi : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
    auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&Phi);
    if (!PhiR || !RecurrenceDescriptor::isFindLastRecurrenceKind(
                     PhiR->getRecurrenceKind()))
      continue;

    // Find the condition for the select/blend.
    VPValue *BackedgeSelect = PhiR->getBackedgeValue();
    VPValue *CondSelect = BackedgeSelect;

    // If there's a header mask, the backedge select will not be the find-last
    // select.
    if (HeaderMask && !match(BackedgeSelect,
                             m_Select(m_Specific(HeaderMask),
                                      m_VPValue(CondSelect), m_Specific(PhiR))))
      llvm_unreachable("expected header mask select");

    VPValue *Cond = nullptr, *Op1 = nullptr, *Op2 = nullptr;

    // If we're matching a blend rather than a select, there should be one
    // incoming value which is the data, then all other incoming values should
    // be the phi.
    auto MatchBlend = [&](VPRecipeBase *R) {
      auto *Blend = dyn_cast<VPBlendRecipe>(R);
      if (!Blend)
        return false;
      assert(!Blend->isNormalized() && "must run before blend normalizaion");
      unsigned NumIncomingDataValues = 0;
      for (unsigned I = 0; I < Blend->getNumIncomingValues(); ++I) {
        VPValue *Incoming = Blend->getIncomingValue(I);
        if (Incoming != PhiR) {
          ++NumIncomingDataValues;
          Cond = Blend->getMask(I);
          Op1 = Incoming;
          Op2 = PhiR;
        }