Repository: sonic-net/sonic-swss
Pull request #3488: FGNHG changes for NHs learned dynamically from BGP
Author: login: manamand2020
Created: 2025-01-29T01:28:47Z | Merged: 2025-04-29T18:13:28Z

<!--
Please make sure you have read and understood the contribution guildlines:
https://github.com/Azure/SONiC/blob/gh-pages/CONTRIBUTING.md

1. Make sure your commit includes a signature generted with `git commit -s`
2. Make sure your commit title follows the correct format: [component]: description
3. Make sure your commit message contains enough details about the change and related tests
4. Make sure your pull request adds related reviewers, asignees, labels

Please also provide the following information in this pull request:
-->

**What I did**
Orchagent changes for Dynamic Fine Grain Nexthop Group as updated in the HLD PR:
https://github.com/sonic-net/SONiC/pull/1823

**Why I did it**
A route prefix points to an ECMP group without all the nexthop members of the group known ahead of time and they are dynamically learnt from the BGP updates. FGNHG current implementation required all nexthops to be known ahead and configured statically.

**How I verified it**
Added FGNHG related DVS tests verified.
Changes were tested with traffic on hardware.

**Details if related**



Diff:
diff --git a/orchagent/fgnhgorch.cpp b/orchagent/fgnhgorch.cpp
index 195f814b781..a78f4096770 100644
--- a/orchagent/fgnhgorch.cpp
+++ b/orchagent/fgnhgorch.cpp
@@ -69,7 +69,7 @@ void FgNhgOrch::update(SubjectType type, void *cntx)
                             {
                                 continue;
                             }
- 
+
                             if (!validNextHopInNextHopGroup(nhk))
                             {
                                 SWSS_LOG_WARN("Failed validNextHopInNextHopGroup for nh %s ip %s",
@@ -142,32 +142,47 @@ bool FgNhgOrch::bake()
 }
 
 /* calculateBankHashBucketStartIndices: generates the hash_bucket_indices for all banks
- * and stores it in fgNhgEntry for the group. 
- * The function will identify the # of next-hops assigned to each bank and 
+ * and stores it in fgNhgEntry for the group.
+ * The function will identify the # of next-hops assigned to each bank and
  * assign the total number of hash buckets for a bank, based on the proportional
- * number of next-hops in the bank. 
- * eg: Bank0: 6 nh, Bank1: 3 nh, total buckets: 30 => 
+ * number of next-hops in the bank.
+ * eg: Bank0: 6 nh, Bank1: 3 nh, total buckets: 30 =>
  *      calculateBankHashBucketStartIndices: Bank0: Bucket# 0-19, Bank1: Bucket# 20-29
  */
 void FgNhgOrch::calculateBankHashBucketStartIndices(FgNhgEntry *fgNhgEntry)
 {
     SWSS_LOG_ENTER();
-    uint32_t num_banks = 0;
+
     vector<uint32_t> memb_per_bank;
-    for (auto nh : fgNhgEntry->next_hops)
+    uint32_t buckets_per_nexthop;
+    uint32_t extra_buckets;
+    uint32_t split_extra_buckets_among_bank;
+    uint32_t num_banks = 0;
+    if (fgNhgEntry->match_mode == FGMatchMode::PREFIX_BASED)
+    {
+        // For prefix_based match_mode there is no bank configuration, so a single bank is used
+        num_banks = 1;
+        memb_per_bank.push_back(fgNhgEntry->max_next_hops); // all nexthops are in bank 0
+        buckets_per_nexthop = fgNhgEntry->real_bucket_size / fgNhgEntry->max_next_hops;
+        extra_buckets = fgNhgEntry->real_bucket_size - (buckets_per_nexthop * fgNhgEntry->max_next_hops);
+    }
+    else
     {
-        while (nh.second.bank + 1 > num_banks)
+        for (auto nh : fgNhgEntry->next_hops)
         {
-            num_banks++;
-            memb_per_bank.push_back(0);
+            while (nh.second.bank + 1 > num_banks)
+            {
+                num_banks++;
+                memb_per_bank.push_back(0);
+            }
+            memb_per_bank[nh.second.bank] = memb_per_bank[nh.second.bank] + 1;
         }
-        memb_per_bank[nh.second.bank] = memb_per_bank[nh.second.bank] + 1;
-    }
 
-    uint32_t buckets_per_nexthop = fgNhgEntry->real_bucket_size/((uint32_t)fgNhgEntry->next_hops.size());
-    uint32_t extra_buckets = fgNhgEntry->real_bucket_size - (buckets_per_nexthop*((uint32_t)fgNhgEntry->next_hops.size()));
-    uint32_t split_extra_buckets_among_bank = extra_buckets/num_banks;
-    extra_buckets = extra_buckets - (split_extra_buckets_among_bank*num_banks);
+        buckets_per_nexthop = fgNhgEntry->real_bucket_size / ((uint32_t)fgNhgEntry->next_hops.size());
+        extra_buckets = fgNhgEntry->real_bucket_size - (buckets_per_nexthop * ((uint32_t)fgNhgEntry->next_hops.size()));
+    }
+    split_extra_buckets_among_bank = extra_buckets / num_banks;
+    extra_buckets = extra_buckets - (split_extra_buckets_among_bank * num_banks);
 
     uint32_t prev_idx = 0;
 
@@ -199,6 +214,7 @@ void FgNhgOrch::calculateBankHashBucketStartIndices(FgNhgEntry *fgNhgEntry)
 void FgNhgOrch::setStateDbRouteEntry(const IpPrefix &ipPrefix, uint32_t index, NextHopKey nextHop)
 {
     SWSS_LOG_ENTER();
+
     string key = ipPrefix.to_string();
     string field = std::to_string(index);
     string value = nextHop.to_string();
@@ -209,7 +225,7 @@ void FgNhgOrch::setStateDbRouteEntry(const IpPrefix &ipPrefix, uint32_t index, N
                   key.c_str(), value.c_str(), index);
 }
 
-bool FgNhgOrch::writeHashBucketChange(FGNextHopGroupEntry *syncd_fg_route_entry, uint32_t index, sai_object_id_t nh_oid,
+bool FgNhgOrch::writeHashBucketChange(FGNextHopGroupEntry *syncd_fg_route_entry, HashBucketIdx index, sai_object_id_t nh_oid,
         const IpPrefix &ipPrefix, NextHopKey nextHop)
 {
     SWSS_LOG_ENTER();
@@ -265,7 +281,7 @@ bool FgNhgOrch::createFineGrainedNextHopGroup(FGNextHopGroupEntry &syncd_fg_rout
 
     if (platform == VS_PLATFORM_SUBSTRING)
     {
-       /* TODO: need implementation for SAI_NEXT_HOP_GROUP_ATTR_REAL_SIZE */ 
+       /* TODO: need implementation for SAI_NEXT_HOP_GROUP_ATTR_REAL_SIZE */
         fgNhgEntry->real_bucket_size = fgNhgEntry->configured_bucket_size;
     }
     else
@@ -289,7 +305,8 @@ bool FgNhgOrch::createFineGrainedNextHopGroup(FGNextHopGroupEntry &syncd_fg_rout
         }
         fgNhgEntry->real_bucket_size = nhg_attr.value.u32;
     }
-
+    // Initialize the vector to store sai next hop group members
+    syncd_fg_route_entry.nhopgroup_members.resize(fgNhgEntry->real_bucket_size, SAI_NULL_OBJECT_ID);
     calculateBankHashBucketStartIndices(fgNhgEntry);
 
     SWSS_LOG_NOTICE("fgnhgorch created next hop group %s of size %d", nextHops.to_string().c_str(), fgNhgEntry->real_bucket_size);
@@ -303,7 +320,7 @@ bool FgNhgOrch::removeFineGrainedNextHopGroup(FGNextHopGroupEntry *syncd_fg_rout
 
     sai_status_t status;
 
-    for (auto nhgm : syncd_fg_route_entry->nhopgroup_members)
+    for (auto &nhgm : syncd_fg_route_entry->nhopgroup_members)
     {
         status = sai_next_hop_group_api->remove_next_hop_group_member(nhgm);
         if (status != SAI_STATUS_SUCCESS)
@@ -382,12 +399,11 @@ bool FgNhgOrch::validNextHopInNextHopGroup(const NextHopKey& nexthop)
                 }
                 fgNhgEntry = member_entry->second;
             }
-            else 
+            else
             {
                 fgNhgEntry = prefix_entry->second;
             }
             std::map<NextHopKey,sai_object_id_t> nhopgroup_members_set;
-
             std::vector<BankMemberChanges> bank_member_changes(
                 fgNhgEntry->hash_bucket_indices.size(), BankMemberChanges());
 
@@ -401,11 +417,19 @@ bool FgNhgOrch::validNextHopInNextHopGroup(const NextHopKey& nexthop)
             {
                 /* Only happens the 1st time when hash_bucket_indices are not inited
                  */
-                for (auto it : fgNhgEntry->next_hops)
+                if (fgNhgEntry->match_mode == FGMatchMode::PREFIX_BASED)
+                {
+                    // create a single bank in case of prefix_based match mode
+                    bank_member_changes.push_back(BankMemberChanges());
+                }
+                else
                 {
-                    while (bank_member_changes.size() <= it.second.bank)
+                    for (auto it : fgNhgEntry->next_hops)
                     {
-                        bank_member_changes.push_back(BankMemberChanges());
+                        while (bank_member_changes.size() <= it.second.bank)
+                        {
+                            bank_member_changes.push_back(BankMemberChanges());
+                        }
                     }
                 }
             }
@@ -436,11 +460,12 @@ bool FgNhgOrch::validNextHopInNextHopGroup(const NextHopKey& nexthop)
             {
                 for (auto active_nh : syncd_fg_route_entry->active_nexthops)
                 {
+                    // include the existing active nhs into the configured bank
                     bank_member_changes[fgNhgEntry->next_hops[active_nh.ip_address].bank].
                         active_nhs.push_back(active_nh);
                 }
 
-                if (!computeAndSetHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, 
+                if (!computeAndSetHashBucketChanges(syncd_fg_route_entry, fgNhgEntry,
                         bank_member_changes, nhopgroup_members_set, route_table.first))
                 {
                     SWSS_LOG_ERROR("Failed to set fine grained next hop %s",
@@ -486,7 +511,7 @@ bool FgNhgOrch::invalidNextHopInNextHopGroup(const NextHopKey& nexthop)
                 }
                 fgNhgEntry = member_entry->second;
             }
-            else 
+            else
             {
                 fgNhgEntry = prefix_entry->second;
             }
@@ -519,7 +544,7 @@ bool FgNhgOrch::invalidNextHopInNextHopGroup(const NextHopKey& nexthop)
             bank_member_changes[fgNhgEntry->next_hops[nexthop.ip_address].bank].
                     nhs_to_del.push_back(nexthop);
 
-            if (!computeAndSetHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, 
+            if (!computeAndSetHashBucketChanges(syncd_fg_route_entry, fgNhgEntry,
                     bank_member_changes, nhopgroup_members_set, route_table.first))
             {
                 SWSS_LOG_ERROR("Failed to set fine grained next hop %s",
@@ -541,30 +566,32 @@ bool FgNhgOrch::invalidNextHopInNextHopGroup(const NextHopKey& nexthop)
 /* setActiveBankHashBucketChanges: Sets hash buckets for active banks and called on a PER bank basis
  * This function deals with a scenario where next-hop changes occurred for the route,
  * and the next-hop change didn't cause an entire bank to go active/inactive.
- * The function uses bank_member_changes to compute the hash buckets to modify, in order to satisfy the next-hop 
+ * The function uses bank_member_changes to compute the hash buckets to modify, in order to satisfy the next-hop
  * availability for the route/neigh.
  * Eg: Prefix A had nhs 1, 2, 3 with 1, 2, 3, being equally distributed over hash buckets
  * 0-59(20 buckets per nh). If there was a nh removal of nh 2, this fn would equally redistribute hash buckets
- * for nh 2 to nh 1 and nh 3. Leading to 30 hash buckets, each, for nh 1 and nh 3, and none for nh 2. 
+ * for nh 2 to nh 1 and nh 3. Leading to 30 hash buckets, each, for nh 1 and nh 3, and none for nh 2.
  * Thereby achieving consistent and layered hashing.
  */
 bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_route_entry, FgNhgEntry *fgNhgEntry,
-        uint32_t bank, uint32_t syncd_bank, std::vector<BankMemberChanges> bank_member_changes, 
+        uint32_t syncd_bank, BankMemberChanges bank_member_change,
         std::map<NextHopKey,sai_object_id_t> &nhopgroup_members_set, const IpPrefix &ipPrefix)
 {
     SWSS_LOG_ENTER();
 
-    BankMemberChanges bank_member_change = bank_member_changes[bank];
     uint32_t add_idx = 0, del_idx = 0;
     FGNextHopGroupMap *bank_fgnhg_map = &(syncd_fg_route_entry->syncd_fgnhg_map[syncd_bank]);
 
+    // Replace hash bucket indices of deleted NHs with added NHs
     while(del_idx < bank_member_change.nhs_to_del.size() &&
             add_idx < bank_member_change.nhs_to_add.size())
     {
+        // get the hash bucket indices for the deleted NHs
         HashBuckets *hash_buckets = &(bank_fgnhg_map->at(bank_member_change.nhs_to_del[del_idx]));
+        // fill the hash bucket indices with the added NHs
         for (uint32_t i = 0; i < hash_buckets->size(); i++)
         {
-            if (!writeHashBucketChange(syncd_fg_route_entry, hash_buckets->at(i), 
+            if (!writeHashBucketChange(syncd_fg_route_entry, hash_buckets->at(i),
                         nhopgroup_members_set[bank_member_change.nhs_to_add[add_idx]],
                         ipPrefix, bank_member_change.nhs_to_add[add_idx]))
             {
@@ -584,7 +611,7 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
     }
 
     /* Given that we resolved add + del on a bank in the above while stmt
-     * We will either have add OR delete left to do, and the logic below 
+     * We will either have add OR delete left to do, and the logic below
      * relies on this fact
      */
     if (del_idx < bank_member_change.nhs_to_del.size())
@@ -593,63 +620,81 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        auto active_nh_list = bank_member_change.active_nhs;
+        bool move_bkt = false;
+        bool remove_nh = false;
 
-
-        while(del_idx < bank_member_change.nhs_to_del.size())
+        auto it = active_nh_list.begin();
+        while (del_idx < bank_member_change.nhs_to_del.size())
         {
             HashBuckets *hash_buckets = &(bank_fgnhg_map->at(bank_member_change.nhs_to_del[del_idx]));
-            for (uint32_t i = 0; i < hash_buckets->size(); i++)
-            {
-                NextHopKey round_robin_nh = bank_member_change.active_nhs[i %
-                    bank_member_change.active_nhs.size()];
 
-                if (!writeHashBucketChange(syncd_fg_route_entry, hash_buckets->at(i), 
-                        nhopgroup_members_set[round_robin_nh], ipPrefix, round_robin_nh))
+            uint32_t bkt_idx = 0;
+            while(bkt_idx < hash_buckets->size())
+            {
+                move_bkt = false; remove_nh = false;
+                // wraparound the active_nh_list if needed
+                if (it == active_nh_list.end())
                 {
-                    return false;
+                    it = active_nh_list.begin();
+                    if (it == active_nh_list.end())
+                    {
+                        // this can neven happen
+                        SWSS_LOG_ERROR("%s Unexpected no more active NHs before adding the %zu buckets, exp_bucket_size(%d)",
+                                       ipPrefix.to_string().c_str(),
+                                       hash_buckets->size(), exp_bucket_size);
+                        return false;
+                    }
                 }
-                bank_fgnhg_map->at(round_robin_nh).push_back(hash_buckets->at(i));
 
                 /* Logic below ensure that # hash buckets assigned to a nh is equalized,
-                 * we could have used simple round robin to reassign hash buckets to 
-                 * other available nhs, but for cases where # hash buckets is not 
+                 * we could have used simple round robin to reassign hash buckets to
+                 * other available nhs, but for cases where # hash buckets is not
                  * divisible by # of nhs, simple round robin can make the hash bucket
                  * distribution non-ideal, thereby nhs can attract unequal traffic */
-                if (num_nhs_with_one_more == 0)
+                if (bank_fgnhg_map->at(*it).size() == exp_bucket_size)
                 {
-                    if (bank_fgnhg_map->at(round_robin_nh).size() == exp_bucket_size)
+                    if (num_nhs_with_one_more == 0)
                     {
-                        SWSS_LOG_INFO("%s reached %d, don't remove more buckets", 
-                                (bank_member_change.active_nhs[i % bank_member_change.active_nhs.size()]).to_string().c_str(), 
-                                exp_bucket_size);
-                        bank_member_change.active_nhs.erase(bank_member_change.active_nhs.begin() + 
-                            (i % bank_member_change.active_nhs.size()));
+
+                        SWSS_LOG_INFO("%s already reached expected bucket size %d, don't add more buckets, bkt_idx(%d)",
+                                      (*it).to_string().c_str(), exp_bucket_size, bkt_idx);
+                        remove_nh = true;
                     }
-                    else if (bank_fgnhg_map->at(round_robin_nh).size() > exp_bucket_size)
+                    else
                     {
-                        SWSS_LOG_WARN("Unexpected bucket size for nh %s, size %zu, exp_size %d",
-                                round_robin_nh.to_string().c_str(), bank_fgnhg_map->at(round_robin_nh).size(),
-                                exp_bucket_size);
+                        num_nhs_with_one_more--;
+                        SWSS_LOG_INFO("%s reached %d, don't add more buckets after this one, num_nhs_with_one_more remaining=%d, bkt_idx(%d)",
+                                      (*it).to_string().c_str(), exp_bucket_size + 1, num_nhs_with_one_more, bkt_idx);
+                        move_bkt = true;
+                        remove_nh = true;
                     }
                 }
                 else
                 {
-                    if (bank_fgnhg_map->at(round_robin_nh).size() == exp_bucket_size +1)
-                    {
+                    SWSS_LOG_INFO("%s has not reached min expected size of %d, keep adding to this nexthop, bkt_idx(%d)",
+                                  (*it).to_string().c_str(), exp_bucket_size, bkt_idx);
+                    move_bkt = true;
+                }
 
-                        SWSS_LOG_INFO("%s reached %d, don't remove more buckets num_nhs_with_one_more %d", 
-                                (bank_member_change.active_nhs[i %bank_member_change.active_nhs.size()]).to_string().c_str(), 
-                                exp_bucket_size +1, num_nhs_with_one_more -1);
-                        bank_member_change.active_nhs.erase(bank_member_change.active_nhs.begin() + 
-                            (i % bank_member_change.active_nhs.size()));
-                        num_nhs_with_one_more--;
-                    }
-                    else if (bank_fgnhg_map->at(round_robin_nh).size() > exp_bucket_size +1)
+                if (move_bkt)
+                {
+                    if (!writeHashBucketChange(syncd_fg_route_entry, hash_buckets->at(bkt_idx),
+                                               nhopgroup_members_set[*it], ipPrefix, *it))
                     {
-                        SWSS_LOG_WARN("Unexpected bucket size for nh %s, size %zu, exp_size %d",
-                                round_robin_nh.to_string().c_str(), bank_fgnhg_map->at(round_robin_nh).size(),
-                                exp_bucket_size + 1);
+                        return false;
                     }
+                    bank_fgnhg_map->at(*it).push_back(hash_buckets->at(bkt_idx));
+                    bkt_idx++;
+                }
+                if (remove_nh)
+                {
+                    // remove the nh from the active list
+                    active_nh_list.erase(it);
+                }
+                else
+                {
+                    it++;
                 }
             }
 
@@ -670,11 +715,16 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
         uint32_t num_nhs_with_eq_to_exp = total_nhs - num_nhs_with_one_more;
         uint32_t add_nh_exp_bucket_size = exp_bucket_size;
 
-        while(add_idx < bank_member_change.nhs_to_add.size())
+
+        auto active_nh_list = bank_member_change.active_nhs;
+        bool move_bkt;
+        bool remove_nh;
+
+        while (add_idx < bank_member_change.nhs_to_add.size())
         {
-            (*bank_fgnhg_map)[bank_member_change.nhs_to_add[add_idx]] = 
-                std::vector<uint32_t>();
-            auto it = bank_member_change.active_nhs.begin();
+            (*bank_fgnhg_map)[bank_member_change.nhs_to_add[add_idx]] =
+                HashBuckets();
+            auto it = active_nh_list.begin();
             if (num_nhs_with_eq_to_exp > 0)
             {
                 num_nhs_with_eq_to_exp--;
@@ -684,77 +734,80 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
                 add_nh_exp_bucket_size = exp_bucket_size + 1;
                 num_nhs_with_one_more--;
             }
-
-            while(bank_fgnhg_map->at(bank_member_change.nhs_to_add[add_idx]).size() != add_nh_exp_bucket_size)
+            for (const auto &nh : active_nh_list)
             {
-                if (it == bank_member_change.active_nhs.end())
-                {
-                    it = bank_member_change.active_nhs.begin();
-                }
-                vector<uint32_t> *map_entry = &(bank_fgnhg_map->at(*it));
-                if ((*map_entry).size() <= 1)
-                {
-                    /* Case where the number of hash buckets for the nh is <= 1 */
-                    SWSS_LOG_WARN("Next-hop %s has %d entries, either number of buckets were less or we hit a bug",
-                            (*it).to_string().c_str(), ((int)(*map_entry).size()));
-                    return false;
-                }
-                else
+                SWSS_LOG_INFO("Active next-hop %s has %zu buckets assigned",
+                              nh.to_string().c_str(), bank_fgnhg_map->at(nh).size());
+            }
+            // add buckets to this nh until HashBuckets size has reached the expected size
+            while (bank_fgnhg_map->at(bank_member_change.nhs_to_add[add_idx]).size() != add_nh_exp_bucket_size)
+            {
+                move_bkt = false;
+                remove_nh = false;
+                if (it == active_nh_list.end())
                 {
-                    uint32_t last_elem = map_entry->at((*map_entry).size() - 1);
-
-                    if (!writeHashBucketChange(syncd_fg_route_entry, last_elem, 
-                        nhopgroup_members_set[bank_member_change.nhs_to_add[add_idx]],
-                        ipPrefix, bank_member_change.nhs_to_add[add_idx]))
-                    {
-                        return false;
-                    }
-
-                    (*bank_fgnhg_map)[bank_member_change.nhs_to_add[add_idx]].push_back(last_elem);
-                    (*map_entry).erase((*map_entry).end() - 1);
+                    it = active_nh_list.begin();
                 }
+                // get the active nh's hashbuckets
+                HashBuckets *map_entry = &(bank_fgnhg_map->at(*it));
                 /* Logic below ensure that # hash buckets assigned to a nh is equalized,
-                 * we could have used simple round robin to reassign hash buckets to 
-                 * other available nhs, but for cases where # hash buckets is not 
+                 * we could have used simple round robin to reassign hash buckets to
+                 * other available nhs, but for cases where # hash buckets is not
                  * divisible by # of nhs, simple round robin can make the hash bucket
                  * distribution non-ideal, thereby nhs can attract unequal traffic */
-                if (num_nhs_with_one_more == 0)
+                if (map_entry->size() > exp_bucket_size + 1)
                 {
-                    if (map_entry->size() == exp_bucket_size)
-                    {
-                        SWSS_LOG_INFO("%s reached %d, don't remove more buckets", it->to_string().c_str(), exp_bucket_size);
-                        it = bank_member_change.active_nhs.erase(it);
-                    }
-                    else if (map_entry->size() < exp_bucket_size)
+                    SWSS_LOG_INFO("Nexthop %s has %zu, continue to remove more buckets after this.",
+                                  it->to_string().c_str(), map_entry->size());
+                    move_bkt=true;
+                }
+                else if (map_entry->size() == exp_bucket_size + 1)
+                {
+                    if (num_nhs_with_one_more == 0)
                     {
-                        SWSS_LOG_WARN("Unexpected bucket size for nh %s, size %zu, exp_size %d",
-                                it->to_string().c_str(), map_entry->size(), exp_bucket_size);
-                        it++;
+                        SWSS_LOG_INFO("Nexthop %s has %zu, take one bucket from it and remove it from the list.",
+                            it->to_string().c_str(), map_entry->size());
+                        remove_nh = true;
+                        move_bkt = true;
                     }
                     else
                     {
-                        it++;
+                        SWSS_LOG_INFO("Nexthop %s with %zu buckets is one of the nexthops with one more, remove it \
+                            from the list. remaining num_nhs_with_one_more = %d",
+                            it->to_string().c_str(), map_entry->size(), num_nhs_with_one_more - 1);
+                        remove_nh = true;
+                        num_nhs_with_one_more--;
                     }
                 }
                 else
                 {
-                    if (map_entry->size() == exp_bucket_size +1)
-                    {
-                        SWSS_LOG_INFO("%s reached %d, don't remove more buckets num_nhs_with_one_more %d", 
-                                it->to_string().c_str(), exp_bucket_size + 1, num_nhs_with_one_more -1);
-                        it = bank_member_change.active_nhs.erase(it);
-                        num_nhs_with_one_more--;
-                    }
-                    else if (map_entry->size() < exp_bucket_size)
-                    {
-                        SWSS_LOG_WARN("Unexpected bucket size for nh %s, size %zu, exp_size %d",
-                                it->to_string().c_str(), map_entry->size(), exp_bucket_size + 1);
-                        it++;
-                    }
-                    else
+                    SWSS_LOG_WARN("Nexthop %s already has %zu, don't remove any buckets but remove this nexthop from the list",
+                        it->to_string().c_str(), map_entry->size());
+                    remove_nh=true;
+                }
+
+                // Replace the active nh's buckets from the end
+                if (move_bkt)
+                {
+                    HashBucketIdx last_elem = map_entry->at((*map_entry).size() - 1);
+                    if (!writeHashBucketChange(syncd_fg_route_entry, last_elem,
+                                               nhopgroup_members_set[bank_member_change.nhs_to_add[add_idx]],
+                                               ipPrefix, bank_member_change.nhs_to_add[add_idx]))
                     {
-                        it++;
+                        return false;
                     }
+
+                    (*bank_fgnhg_map)[bank_member_change.nhs_to_add[add_idx]].push_back(last_elem);
+                    (*map_entry).erase((*map_entry).end() - 1);
+                }
+
+                // If done with current nexthop from the active_nh_list, then remove it. Else simply move to next nexthop
+                // in the active_nh_list
+                if (remove_nh)
+                {
+                    active_nh_list.erase(it);
+                } else {
+                    it++;
                 }
             }
             syncd_fg_route_entry->active_nexthops.insert(bank_member_change.nhs_to_add[add_idx]);
@@ -861,10 +914,10 @@ bool FgNhgOrch::setInactiveBankToNextAvailableActiveBank(FGNextHopGroupEntry *sy
  * Eg: Lets assume prefix A had nhs 1, 2, 3, 4, 5, 6 with nhs being equally distributed over hash buckets
  * 0-59(10 per nh). Now there was a nh deletion of 1, 2, 3 which constituted bank 0(4, 5, 6 constituted bank 1)
  * This function will identify that all of bank 0's nh are down and re-assign all the hash buckets(0-29) for these nhs to
- * nhs from bank 1, along with making local struct changes to track this for future route/neigh changes. 
+ * nhs from bank 1, along with making local struct changes to track this for future route/neigh changes.
  */
 bool FgNhgOrch::setInactiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_route_entry, FgNhgEntry *fgNhgEntry,
-        uint32_t bank,std::vector<BankMemberChanges> &bank_member_changes, 
+        uint32_t bank,std::vector<BankMemberChanges> &bank_member_changes,
         std::map<NextHopKey,sai_object_id_t> &nhopgroup_members_set, const IpPrefix &ipPrefix)
 {
     SWSS_LOG_ENTER();
@@ -872,6 +925,7 @@ bool FgNhgOrch::setInactiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_r
     if (bank_member_changes[bank].nhs_to_add.size() > 0)
     {
         /* Previously inactive bank now transitions to active */
+        SWSS_LOG_INFO("Previously inactive bank now transitions to active bank %u", bank);
         syncd_fg_route_entry->syncd_fgnhg_map[bank].clear();
         for (uint32_t i = fgNhgEntry->hash_bucket_indices[bank].start_index;
                 i <= fgNhgEntry->hash_bucket_indices[bank].end_index; i++)
@@ -879,7 +933,7 @@ bool FgNhgOrch::setInactiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_r
             NextHopKey bank_nh_memb = bank_member_changes[bank].
                 nhs_to_add[i % bank_member_changes[bank].nhs_to_add.size()];
 
-            if (!writeHashBucketChange(syncd_fg_route_entry, i, 
+            if (!writeHashBucketChange(syncd_fg_route_entry, i,
                   nhopgroup_members_set[bank_nh_memb], ipPrefix, bank_nh_memb))
             {
                 return false;
@@ -890,12 +944,13 @@ bool FgNhgOrch::setInactiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_r
         }
         syncd_fg_route_entry->inactive_to_active_map[bank] = bank;
 
-        SWSS_LOG_NOTICE("Bank# %d of FG next-hops is up for prefix %s", 
+        SWSS_LOG_NOTICE("Bank# %d of FG next-hops is up for prefix %s",
                 bank, ipPrefix.to_string().c_str());
     }
     else if (bank_member_changes[bank].nhs_to_del.size() > 0)
     {
         /* Previously active bank now transitions to inactive */
+        SWSS_LOG_INFO("Previously active bank now transitions to inactive bank %u", bank);
         if (!setInactiveBankToNextAvailableActiveBank(syncd_fg_route_entry, fgNhgEntry,
                     bank, bank_member_changes, nhopgroup_members_set, ipPrefix))
         {
@@ -908,28 +963,27 @@ bool FgNhgOrch::setInactiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_r
             syncd_fg_route_entry->active_nexthops.erase(memb);
         }
 
-        SWSS_LOG_NOTICE("Bank# %d of FG next-hops is down for prefix %s", bank, 
+        SWSS_LOG_NOTICE("Bank# %d of FG next-hops is down for prefix %s", bank,
                 ipPrefix.to_string().c_str());
     }
     else
     {
         /* Previously inactive bank remains inactive */
         uint32_t active_bank = syncd_fg_route_entry->inactive_to_active_map[bank];
+        SWSS_LOG_INFO("Previously inactive bank remains inactive bank %u active bank %u", bank, active_bank);
         if (bank_member_changes[active_bank].active_nhs.size() == 0)
         {
             if (!setInactiveBankToNextAvailableActiveBank(syncd_fg_route_entry, fgNhgEntry,
                         bank, bank_member_changes, nhopgroup_members_set, ipPrefix))
             {
-                SWSS_LOG_INFO("Failed to map to active_bank and set nh in SAI");
                 return false;
             }
         }
         else
         {
-            if (!setActiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, 
-                active_bank, bank, bank_member_changes, nhopgroup_members_set, ipPrefix))
+            if (!setActiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry,
+                bank, bank_member_changes[active_bank], nhopgroup_members_set, ipPrefix))
             {
-                SWSS_LOG_INFO("Failed setActiveBankHashBucketChanges");
                 return false;
             }
         }
@@ -938,8 +992,8 @@ bool FgNhgOrch::setInactiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_r
 }
 
 
-bool FgNhgOrch::computeAndSetHashBucketChanges(FGNextHopGroupEntry *syncd_fg_route_entry, 
-        FgNhgEntry *fgNhgEntry, std::vector<BankMemberChanges> &bank_member_changes, 
+bool FgNhgOrch::computeAndSetHashBucketChanges(FGNextHopGroupEntry *syncd_fg_route_entry,
+        FgNhgEntry *fgNhgEntry, std::vector<BankMemberChanges> &bank_member_changes,
         std::map<NextHopKey,sai_object_id_t> &nhopgroup_members_set,
         const IpPrefix &ipPrefix)
 {
@@ -951,20 +1005,21 @@ bool FgNhgOrch::computeAndSetHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
                 (bank_member_changes[bank_idx].nhs_to_add.size() != 0 &&
                  bank_member_changes[bank_idx].nhs_to_del.size() != 0))
         {
+            SWSS_LOG_INFO("active nhs in bank %u", bank_idx);
             /* Active bank is is determined by there being active nhs on the bank OR
              * an edge case where all active_nhs went down(nhs_to_del > 0) BUT
-             * simultaneously, nhs were added(nhs_to_add > 0). 
+             * simultaneously, nhs were added(nhs_to_add > 0).
              * Route this to fn which deals with active banks
              */
-            if (!setActiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, 
-                        bank_idx, bank_idx, bank_member_changes, nhopgroup_members_set, ipPrefix))
+            if (!setActiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry,
+                        bank_idx, bank_member_changes[bank_idx], nhopgroup_members_set, ipPrefix))
             {
                 return false;
             }
         }
         else
         {
-            if (!setInactiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, 
+            if (!setInactiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry,
                         bank_idx, bank_member_changes, nhopgroup_members_set, ipPrefix))
             {
                 return false;
@@ -976,126 +1031,166 @@ bool FgNhgOrch::computeAndSetHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
 }
 
 
-bool FgNhgOrch::setNewNhgMembers(FGNextHopGroupEntry &syncd_fg_route_entry, FgNhgEntry *fgNhgEntry, 
+bool FgNhgOrch::setNewNhgMembers(FGNextHopGroupEntry &syncd_fg_route_entry, FgNhgEntry *fgNhgEntry,
         std::vector<BankMemberChanges> &bank_member_changes, std::map<NextHopKey,sai_object_id_t> &nhopgroup_members_set,
         const IpPrefix &ipPrefix)
 {
     SWSS_LOG_ENTER();
 
-    sai_status_t status;
-    bool isWarmReboot = false;
-    auto nexthopsMap = m_recoveryMap.find(ipPrefix.to_string());
-    for (uint32_t i = 0; i < fgNhgEntry->hash_bucket_indices.size(); i++) 
+    auto num_banks = fgNhgEntry->hash_bucket_indices.size();
+    std::vector<uint32_t> active_banks;
+    std::vector<uint32_t> inactive_banks;
+
+    for (uint32_t i = 0; i < num_banks; i++)
     {
-        uint32_t bank = i;
-        syncd_fg_route_entry.inactive_to_active_map[bank] = bank;
+        if (bank_member_changes[i].nhs_to_add.size() != 0)
+        {
+            active_banks.push_back(i);
+            SWSS_LOG_INFO("active bank %d num nhs %zu", i, bank_member_changes[i].nhs_to_add.size());
+        }
+        else
+        {
+            inactive_banks.push_back(i);
+            SWSS_LOG_INFO("inactive bank %d", i);
+        }
+
+        // add a new fgnh group map if this is a new bank
         if (i + 1 > syncd_fg_route_entry.syncd_fgnhg_map.size())
         {
             syncd_fg_route_entry.syncd_fgnhg_map.push_back(FGNextHopGroupMap());
         }
+    }
 
-        if (bank_member_changes[i].nhs_to_add.size() == 0)
-        {
-            /* Case where bank is empty */
-            for (uint32_t active_bank = 0; active_bank < bank_member_changes.size(); active_bank++)
-            {
-                if (bank_member_changes[active_bank].nhs_to_add.size() != 0)
-                {
-                    bank = active_bank;
-                    syncd_fg_route_entry.inactive_to_active_map[i] = active_bank;
-                    break;
-                }
-            }
-            SWSS_LOG_NOTICE("Bank# %d of FG next-hops is down for prefix %s", 
-                    i, ipPrefix.to_string().c_str());
-        } 
 
-        if (bank_member_changes[bank].nhs_to_add.size() == 0)
+    /* Case where all banks are empty, we let retry logic(upon rv false) take care of this scenario */
+    if (active_banks.size() == 0)
+    {
+        SWSS_LOG_INFO("Found no next-hops to add, skipping");
+        return false;
+    }
+    // spray the nhs of active banks into active bank's buckets
+    for (auto active_bank : active_banks)
+    {
+        syncd_fg_route_entry.inactive_to_active_map[active_bank] = active_bank;
+        if (!sprayBankNhgMembers(syncd_fg_route_entry, ipPrefix,
+                fgNhgEntry->hash_bucket_indices[active_bank], fgNhgEntry,
+                active_bank, bank_member_changes[active_bank],
+                nhopgroup_members_set))
         {
-            /* Case where all banks are empty, we let retry logic(upon rv false) take care of this scenario */
-            SWSS_LOG_INFO("Found no next-hops to add, skipping");
             return false;
         }
+    }
 
-        // recover state before warm reboot
-        if (nexthopsMap != m_recoveryMap.end())
+    // spray the nhs of active banks into inactive bank's buckets
+    for (uint32_t i = 0; i < inactive_banks.size(); i++)
+    {
+        auto active_bank = active_banks[i % active_banks.size()];
+        syncd_fg_route_entry.inactive_to_active_map[inactive_banks[i]] =
+            active_bank;
+        if(!sprayBankNhgMembers(syncd_fg_route_entry, ipPrefix,
+                fgNhgEntry->hash_bucket_indices[inactive_banks[i]], fgNhgEntry,
+                inactive_banks[i], bank_member_changes[active_bank],
+                nhopgroup_members_set))
         {
-            isWarmReboot = true;
+            return false;
         }
+        SWSS_LOG_NOTICE("Bank# %d of FG next-hops is down for prefix %s",
+                inactive_banks[i], ipPrefix.to_string().c_str());
+    }
+
+    auto nexthopsMap = m_recoveryMap.find(ipPrefix.to_string());
+    if (nexthopsMap != m_recoveryMap.end()) {
+        m_recoveryMap.erase(nexthopsMap);
+    }
+
+    syncd_fg_route_entry.points_to_rif = false;
+
+    return true;
+}
 
-        SWSS_LOG_INFO("Warm reboot is set to %d", isWarmReboot);
+bool FgNhgOrch::sprayBankNhgMembers(FGNextHopGroupEntry &syncd_fg_route_entry, const IpPrefix &ipPrefix,
+        BankIndexRange hash_idx_range, FgNhgEntry *fgNhgEntry,
+        uint32_t bank, BankMemberChanges &bank_member_change,
+        std::map<NextHopKey,sai_object_id_t> &nhopgroup_members_set)
+{
+    sai_status_t status;
+    bool isWarmReboot = false;
+    auto nexthopsMap = m_recoveryMap.find(ipPrefix.to_string());
+
+    SWSS_LOG_ENTER();
 
-        for (uint32_t j = fgNhgEntry->hash_bucket_indices[i].start_index;
-                j <= fgNhgEntry->hash_bucket_indices[i].end_index; j++)
+    // recover state before warm reboot
+    if (nexthopsMap != m_recoveryMap.end())
+    {
+        isWarmReboot = true;
+    }
+
+    SWSS_LOG_INFO("Warm reboot is set to %d, bank %d", isWarmReboot, bank);
+
+    // fill the hash idx range with the nhs
+    for (uint32_t bucket_idx = hash_idx_range.start_index;
+            bucket_idx <= hash_idx_range.end_index; bucket_idx++)
+    {
+        NextHopKey nh_memb_key;
+        if (isWarmReboot)
         {
-            NextHopKey bank_nh_memb;
-            if (isWarmReboot)
+            nh_memb_key = nexthopsMap->second[bucket_idx];
+            SWSS_LOG_INFO("Recovering nexthop %s with bucket_idx %d", nh_memb_key.ip_address.to_string().c_str(), bucket_idx);
+            // case nhps in bank are all down
+            if (fgNhgEntry->next_hops[nh_memb_key.ip_address].bank != bank)
             {
-                bank_nh_memb = nexthopsMap->second[j];
-                SWSS_LOG_INFO("Recovering nexthop %s with bucket %d", bank_nh_memb.ip_address.to_string().c_str(), j);
-                // case nhps in bank are all down
-                if (fgNhgEntry->next_hops[bank_nh_memb.ip_address].bank != i)
-                {
-                    syncd_fg_route_entry.inactive_to_active_map[i] = fgNhgEntry->next_hops[bank_nh_memb.ip_address].bank;
-                }
-            }
-            else
-            {
-                bank_nh_memb = bank_member_changes[bank].nhs_to_add[j %
-                    bank_member_changes[bank].nhs_to_add.size()];
+                syncd_fg_route_entry.inactive_to_active_map[bank] = fgNhgEntry->next_hops[nh_memb_key.ip_address].bank;
             }
+        }
+        else
+        {
+            nh_memb_key = bank_member_change.nhs_to_add[bucket_idx %
+                bank_member_change.nhs_to_add.size()];
+        }
 
-            // Create a next hop group member
-            sai_attribute_t nhgm_attr;
-            vector<sai_attribute_t> nhgm_attrs;
-            nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID;
-            nhgm_attr.value.oid = syncd_fg_route_entry.next_hop_group_id;
-            nhgm_attrs.push_back(nhgm_attr);
+        // Create a next hop group member
+        sai_attribute_t nhgm_attr;
+        vector<sai_attribute_t> nhgm_attrs;
+        nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID;
+        nhgm_attr.value.oid = syncd_fg_route_entry.next_hop_group_id;
+        nhgm_attrs.push_back(nhgm_attr);
 
-            nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID;
-            nhgm_attr.value.oid = nhopgroup_members_set[bank_nh_memb];
-            nhgm_attrs.push_back(nhgm_attr);
+        nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID;
+        nhgm_attr.value.oid = nhopgroup_members_set[nh_memb_key];
+        nhgm_attrs.push_back(nhgm_attr);
 
-            nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_INDEX;
-            nhgm_attr.value.s32 = j;
-            nhgm_attrs.push_back(nhgm_attr);
+        nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_INDEX;
+        nhgm_attr.value.s32 = bucket_idx;
+        nhgm_attrs.push_back(nhgm_attr);
 
-            sai_object_id_t next_hop_group_member_id;
-            status = sai_next_hop_group_api->create_next_hop_group_member(
-                                                              &next_hop_group_member_id,
-                                                              gSwitchId,
-                                                              (uint32_t)nhgm_attrs.size(),
-                                                              nhgm_attrs.data());
-            if (status != SAI_STATUS_SUCCESS)
+        sai_object_id_t next_hop_group_member_id;
+        status = sai_next_hop_group_api->create_next_hop_group_member(
+                                                          &next_hop_group_member_id,
+                                                          gSwitchId,
+                                                          (uint32_t)nhgm_attrs.size(),
+                                                          nhgm_attrs.data());
+        if (status != SAI_STATUS_SUCCESS)
+        {
+            SWSS_LOG_ERROR("Failed to create next hop group %" PRIx64 " member %" PRIx64 ": %d",
+               syncd_fg_route_entry.next_hop_group_id, next_hop_group_member_id, status);
+            if (!removeFineGrainedNextHopGroup(&syncd_fg_route_entry))
             {
-                SWSS_LOG_ERROR("Failed to create next hop group %" PRIx64 " member %" PRIx64 ": %d",
-                   syncd_fg_route_entry.next_hop_group_id, next_hop_group_member_id, status);
-                
-                if (!removeFineGrainedNextHopGroup(&syncd_fg_route_entry))
-                {
-                    SWSS_LOG_ERROR("Failed to clean-up after next-hop member creation failure");
-                }
-                
-                task_process_status handle_status = handleSaiCreateStatus(SAI_API_NEXT_HOP_GROUP, status);
-                if (handle_status != task_success)
-                {
-                    return parseHandleSaiStatusFailure(handle_status);
-                }
+                SWSS_LOG_ERROR("Failed to clean-up after next-hop member creation failure");
             }
 
-            setStateDbRouteEntry(ipPrefix, j, bank_nh_memb);
-            syncd_fg_route_entry.syncd_fgnhg_map[i][bank_nh_memb].push_back(j);
-            syncd_fg_route_entry.active_nexthops.insert(bank_nh_memb);
-            syncd_fg_route_entry.nhopgroup_members.push_back(next_hop_group_member_id);
-            gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER);
+            task_process_status handle_status = handleSaiCreateStatus(SAI_API_NEXT_HOP_GROUP, status);
+            if (handle_status != task_success)
+            {
+                return parseHandleSaiStatusFailure(handle_status);
+            }
         }
-    }
 
-    if (isWarmReboot)
-    {
-        m_recoveryMap.erase(nexthopsMap);
+        setStateDbRouteEntry(ipPrefix, bucket_idx, nh_memb_key);
+        syncd_fg_route_entry.syncd_fgnhg_map[bank][nh_memb_key].push_back(bucket_idx);
+        syncd_fg_route_entry.active_nexthops.insert(nh_memb_key);
+        syncd_fg_route_entry.nhopgroup_members[bucket_idx] = next_hop_group_member_id;
+        gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER);
     }
-    syncd_fg_route_entry.points_to_rif = false;
 
     return true;
 }
@@ -1104,9 +1199,11 @@ bool FgNhgOrch::setNewNhgMembers(FGNextHopGroupEntry &syncd_fg_route_entry, FgNh
 bool FgNhgOrch::isRouteFineGrained(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const NextHopGroupKey &nextHops)
 {
     SWSS_LOG_ENTER();
- 
+
     if (!isFineGrainedConfigured || (vrf_id != gVirtualRouterId))
     {
+        SWSS_LOG_DEBUG("Route %s:%s vrf %" PRIx64 " default_vrf %" PRIx64 " NOT fine grained ECMP",
+                        ipPrefix.to_string().c_str(), nextHops.to_string().c_str(), vrf_id, gVirtualRouterId);
         return false;
     }
 
@@ -1139,13 +1236,15 @@ bool FgNhgOrch::isRouteFineGrained(sai_object_id_t vrf_id, const IpPrefix &ipPre
                  */
                 if (fgNhgEntry != member_entry->second)
                 {
-                    SWSS_LOG_INFO("FG nh found across different FG_NH groups: %s expected %s, actual %s", 
+                    SWSS_LOG_INFO("FG nh found across different FG_NH groups: %s expected %s, actual %s",
                         nhk.to_string().c_str(), fgNhgEntry->fg_nhg_name.c_str(), member_entry->second->fg_nhg_name.c_str());
                     return false;
                 }
             }
         }
     }
+    SWSS_LOG_DEBUG("Route %s:%s vrf %" PRIx64 " default_vrf %" PRIx64 " IS fine grained ECMP",
+                    ipPrefix.to_string().c_str(), nextHops.to_string().c_str(), vrf_id, gVirtualRouterId);
     return true;
 }
 
@@ -1210,6 +1309,15 @@ bool FgNhgOrch::setFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const
         return true;
     }
 
+    if (!fgNhgEntry)
+    {
+        SWSS_LOG_ERROR("fgNhgOrch got a route addition %s:%s with %s",
+                ipPrefix.to_string().c_str(), nextHops.to_string().c_str(),
+                (prefix_entry == m_fgNhgPrefixes.end()) ? " no prefix and FG ECMP nexthop entries" :
+                "no FG ECMP entry for the prefix");
+        return true;
+    }
+
     if (m_syncdFGRouteTables.find(vrf_id) == m_syncdFGRouteTables.end())
     {
         m_syncdFGRouteTables.emplace(vrf_id, FGRouteTable());
@@ -1227,11 +1335,72 @@ bool FgNhgOrch::setFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const
     {
         /* Only happens the 1st time when hash_bucket_indices are not initialized
          */
-        for (auto it : fgNhgEntry->next_hops)
+        if (fgNhgEntry->match_mode == FGMatchMode::PREFIX_BASED)
+        {
+            bank_member_changes.resize(1, BankMemberChanges()); // prefix_based match_mode supports single bank
+        }
+        else
+        {
+            for (auto it : fgNhgEntry->next_hops)
+            {
+                while (bank_member_changes.size() <= it.second.bank)
+                {
+                    bank_member_changes.push_back(BankMemberChanges());
+                }
+            }
+        }
+    }
+
+    // if match_mode==PREFIX_BASED, then initialize fgNhgEntry->next_hops with the list of IP addresses in nextHops
+    if (fgNhgEntry->match_mode == FGMatchMode::PREFIX_BASED)
+    {
+        for (NextHopKey nhk : next_hop_set)
         {
-            while (bank_member_changes.size() <= it.second.bank)
+            if (fgNhgEntry->next_hops.find(nhk.ip_address) == fgNhgEntry->next_hops.end())
             {
-                bank_member_changes.push_back(BankMemberChanges());
+                if(fgNhgEntry->next_hops.size() >= fgNhgEntry->max_next_hops)
+                {
+                    SWSS_LOG_WARN("Next-hop %s exceeds max_next_hops %d for prefix %s, skipping",
+                            nhk.to_string().c_str(), fgNhgEntry->max_next_hops, ipPrefix.to_string().c_str());
+                    continue;
+                }
+                FGNextHopInfo fg_nh_info = {0, "", LINK_DOWN}; // prefix_based match_mode uses single bank 0
+                // find link for the next hop
+                if (m_neighOrch->hasNextHop(nhk))
+                {
+                    Port p;
+                    if (gPortsOrch->getPort(nhk.alias, p))
+                    {
+                        // track Physical links
+                        if (p.m_type == Port::PHY)
+                        {
+                            fg_nh_info.link = p.m_alias;
+                            if (p.m_oper_status == SAI_PORT_OPER_STATUS_UP)
+                            {
+                                fg_nh_info.link_oper_state = LINK_UP;
+                            }
+                            auto link_info = fgNhgEntry->links.find(p.m_alias);
+                            if (link_info != fgNhgEntry->links.end())
+                            {
+                                // add nhk.ip_address to the list of next-hops for the link
+                                fgNhgEntry->links[p.m_alias].push_back(nhk.ip_address);
+                            }
+                            else {
+                                // create a new list with nhk.ip_address
+                                fgNhgEntry->links[p.m_alias] = {nhk.ip_address};
+                            }
+                        }
+                    }
+                    else
+                    {
+                        SWSS_LOG_WARN("Next-hop %s alias %s prefix %s Link not found",
+                                nhk.ip_address.to_string().c_str(), nhk.alias.c_str(),
+                                ipPrefix.to_string().c_str());
+                    }
+                }
+                fgNhgEntry->next_hops[nhk.ip_address] = fg_nh_info;
+                SWSS_LOG_INFO("Next-hop %s alias %s added to Fine Grained next-hop group member list for prefix %s",
+                        nhk.ip_address.to_string().c_str(), nhk.alias.c_str(), ipPrefix.to_string().c_str());
             }
         }
     }
@@ -1273,10 +1442,10 @@ bool FgNhgOrch::setFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const
                 nhs_to_add.push_back(nhk);
             next_hop_to_add = true;
         }
-        else 
+        else
         {
             FGNextHopGroupEntry *syncd_fg_route_entry = &(syncd_fg_route_entry_it->second);
-            if (syncd_fg_route_entry->active_nexthops.find(nhk) == 
+            if (syncd_fg_route_entry->active_nexthops.find(nhk) ==
                 syncd_fg_route_entry->active_nexthops.end())
             {
                 bank_member_changes[fgNhgEntry->next_hops[nhk.ip_address].bank].
@@ -1327,7 +1496,7 @@ bool FgNhgOrch::setFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const
                 }
             }
 
-            if (!computeAndSetHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, bank_member_changes, 
+            if (!computeAndSetHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, bank_member_changes,
                     nhopgroup_members_set, ipPrefix))
             {
                 return false;
@@ -1367,7 +1536,7 @@ bool FgNhgOrch::setFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const
 
         m_syncdFGRouteTables[vrf_id][ipPrefix] = syncd_fg_route_entry;
     }
-    m_syncdFGRouteTables[vrf_id][ipPrefix].nhg_key = nextHops; 
+    m_syncdFGRouteTables[vrf_id][ipPrefix].nhg_key = nextHops;
 
     for (uint32_t bank_idx = 0; bank_idx < bank_member_changes.size(); bank_idx++)
     {
@@ -1403,7 +1572,7 @@ bool FgNhgOrch::removeFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix)
     auto it_route_table = m_syncdFGRouteTables.find(vrf_id);
     if (it_route_table == m_syncdFGRouteTables.end())
     {
-        SWSS_LOG_INFO("No route table found for %s, vrf_id 0x%" PRIx64, 
+        SWSS_LOG_INFO("No route table found for %s, vrf_id 0x%" PRIx64,
                 ipPrefix.to_string().c_str(), vrf_id);
         return true;
     }
@@ -1486,7 +1655,7 @@ void FgNhgOrch::cleanupIpInLinkToIpMap(const string &link, const IpAddress &ip,
             SWSS_LOG_WARN("Unexpected case where structs are out of sync for %s",
                     link.c_str());
             return;
-        } 
+        }
         for (auto ip_it = begin(link_entry->second); ip_it != end(link_entry->second); ip_it++)
         {
             if (*ip_it == ip)
@@ -1504,9 +1673,10 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t)
     SWSS_LOG_ENTER();
     string op = kfvOp(t);
     string key = kfvKey(t);
-    string fg_nhg_name = key; 
+    string fg_nhg_name = key;
     auto fgNhg_entry = m_FgNhgs.find(fg_nhg_name);
-    FGMatchMode match_mode = ROUTE_BASED;
+    FGMatchMode match_mode = FGMatchMode::ROUTE_BASED;
+    uint32_t max_next_hops = 0;
 
     if (op == SET_COMMAND)
     {
@@ -1522,7 +1692,11 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t)
             {
                 if (fvValue(i) == "nexthop-based")
                 {
-                    match_mode = NEXTHOP_BASED;
+                    match_mode = FGMatchMode::NEXTHOP_BASED;
+                }
+                else if (fvValue(i) == "prefix-based")
+                {
+                    match_mode = FGMatchMode::PREFIX_BASED;
                 }
                 else if (fvValue(i) != "route-based")
                 {
@@ -1530,6 +1704,17 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t)
                                     fvValue(i).c_str());
                 }
             }
+            else if (fvField(i) == "max_next_hops")
+            {
+
+                max_next_hops = stoi(fvValue(i));
+            }
+        }
+
+        // Generate an error if num_next_hops is not set for prefix_based match_mode
+        if (match_mode == FGMatchMode::PREFIX_BASED && max_next_hops == 0)
+        {
+            SWSS_LOG_ERROR("Received match_mode==prefix_based with max_next_hops 0, not a supported combination");
         }
 
         if (bucket_size == 0)
@@ -1538,7 +1723,7 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t)
             return true;
         }
 
-        if (fgNhg_entry != m_FgNhgs.end()) 
+        if (fgNhg_entry != m_FgNhgs.end())
         {
             SWSS_LOG_WARN("FG_NHG %s already exists, ignoring", fg_nhg_name.c_str());
         }
@@ -1548,8 +1733,9 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t)
             fgNhgEntry.configured_bucket_size = bucket_size;
             fgNhgEntry.fg_nhg_name = fg_nhg_name;
             fgNhgEntry.match_mode = match_mode;
-            SWSS_LOG_NOTICE("Added new FG_NHG entry with bucket_size %d, match_mode: %'" PRIu8, 
-                    bucket_size, match_mode);
+            fgNhgEntry.max_next_hops = max_next_hops;
+            SWSS_LOG_NOTICE("Added new FG_NHG entry %s with bucket_size %d, match_mode: %'" PRIu8,
+                    fg_nhg_name.c_str(), bucket_size, match_mode);
             isFineGrainedConfigured = true;
             m_FgNhgs[fg_nhg_name] = fgNhgEntry;
         }
@@ -1561,7 +1747,7 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t)
             SWSS_LOG_INFO("Received delete call for non-existent entry %s",
                     fg_nhg_name.c_str());
         }
-        else 
+        else
         {
             /* Check if there are no child objects associated prior to deleting */
             if (fgNhg_entry->second.prefixes.size() == 0 && fgNhg_entry->second.next_hops.size() == 0)
@@ -1572,9 +1758,22 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t)
             }
             else
             {
-                SWSS_LOG_INFO("Child Prefix/Member entries are still associated with this FG_NHG %s", 
+                /* in case of prefix-based match mode, members are not deleted by config, so delete them here */
+                if (fgNhg_entry->second.prefixes.size() == 0 &&
+                        fgNhg_entry->second.match_mode == FGMatchMode::PREFIX_BASED &&
+                        fgNhg_entry->second.next_hops.size() != 0)
+                {
+                    fgNhg_entry->second.next_hops.clear();
+                    m_FgNhgs.erase(fgNhg_entry);
+                    SWSS_LOG_INFO("Clearing dynamic NH Member entries associated with this FG_NHG %s",
+                         fg_nhg_name.c_str());
+                }
+                else
+                {
+                    SWSS_LOG_INFO("Child Prefix/Member entries are still associated with this FG_NHG %s",
                         fg_nhg_name.c_str());
-                return false;
+                    return false;
+                }
             }
             if (m_FgNhgs.size() == 0)
             {
@@ -1623,6 +1822,21 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t)
             return false;
         }
 
+        // In the prefix-based match mode, only 1 FG_NHG_PREFIX is allowed for each FG_NHG since we can't
+        // handle the case where multiple prefixes are attached to the same FG_NHG and we receive routes for
+        // these prefixes with disjoint nexthops. If ip_prefix doesn't match with the existing prefixe in
+        // the fgNhgEntry, then return false.
+        if (fgNhg_entry->second.match_mode == FGMatchMode::PREFIX_BASED)
+        {
+            if (fgNhg_entry->second.prefixes.size() != 0)
+            {
+                SWSS_LOG_NOTICE("FG_NHG %s already has a prefix %s, ignoring addition of new prefix %s",
+                                 fg_nhg_name.c_str(), fgNhg_entry->second.prefixes[0].to_string().c_str(),
+                                 ip_prefix.to_string().c_str());
+                return true;
+            }
+        }
+
         if (fgNhg_entry->second.match_mode == NEXTHOP_BASED)
         {
             SWSS_LOG_NOTICE("FG_NHG %s is configured as nexthop_based: FG_NHG_PREFIX is a no-op",
@@ -1664,7 +1878,8 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t)
             }
             else
             {
-                SWSS_LOG_INFO("Route exists in routeorch, and APP_DB route was deleted, waiting for routeorch delete to complete");
+                SWSS_LOG_INFO("Route(%s) ADD exists in routeorch, and APP_DB route was deleted, waiting for routeorch delete to complete",
+                        ip_prefix.to_string().c_str());
                 return false;
             }
         }
@@ -1686,14 +1901,16 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t)
                     m_syncdFGRouteTables.at(vrf_id).find(ip_prefix) != m_syncdFGRouteTables.at(vrf_id).end())
         {
             nhg = m_syncdFGRouteTables.at(vrf_id).at(ip_prefix).nhg_key;
-        } 
+        }
 
         auto delCache = m_fgPrefixDelCache.find(ip_prefix);
         if (delCache == m_fgPrefixDelCache.end())
         {
             if (nhg.getSize() == 0)
             {
-                SWSS_LOG_INFO("Route does not exist in fgnhgorch, proceed with deletion of local structures");
+                SWSS_LOG_INFO("Route(%s) does not exist in fgnhgorch, proceed with deletion of local structures",
+                        ip_prefix.to_string().c_str());
+
                 for (uint32_t i = 0; i < fgNhg_entry->prefixes.size(); i++)
                 {
                     if(fgNhg_entry->prefixes[i] == ip_prefix)
@@ -1703,11 +1920,11 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t)
                     }
                 }
 
-                m_fgNhgPrefixes.erase(ip_prefix); 
+                m_fgNhgPrefixes.erase(ip_prefix);
             }
             else
             {
-                SWSS_LOG_INFO("Route exists in fgNhgOrch, deleting from APP_DB");
+                SWSS_LOG_INFO("Route(%s) exists in fgNhgOrch, deleting from APP_DB", ip_prefix.to_string().c_str());
                 m_fgPrefixDelCache[ip_prefix] = nhg;
                 m_routeTable.del(ip_prefix.to_string());
                 return false;
@@ -1718,7 +1935,7 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t)
             if (nhg.getSize() == 0)
             {
                 /* Case where fgnhgorch route entry was present and the route delete was completed */
-                SWSS_LOG_INFO("Route removed in fgNhgOrch, now do an APP_DB addition");
+                SWSS_LOG_INFO("Route(%s) removed in fgNhgOrch, now do an APP_DB addition", ip_prefix.to_string().c_str());
                 for (uint32_t i = 0; i < fgNhg_entry->prefixes.size(); i++)
                 {
                     if (fgNhg_entry->prefixes[i] == ip_prefix)
@@ -1727,14 +1944,16 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t)
                         break;
                     }
                 }
-                m_fgNhgPrefixes.erase(ip_prefix); 
+                m_fgNhgPrefixes.erase(ip_prefix);
 
                 m_routeTable.set(ip_prefix.to_string(), generateRouteTableFromNhgKey(delCache->second));
+                m_fgPrefixDelCache.erase(delCache);
                 SWSS_LOG_INFO("Perform APP_DB addition with prefix %s", ip_prefix.to_string().c_str());
             }
             else
             {
-                SWSS_LOG_INFO("Route exists in fgNhgOrch, and APP_DB route was deleted, waiting for fgNhgOrch delete to complete");
+                SWSS_LOG_INFO("Route(%s) DEL exists in fgNhgOrch size(%zu), and APP_DB route was deleted, waiting for fgNhgOrch delete to complete!!",
+                        ip_prefix.to_string().c_str(), nhg.getSize());
                 return false;
             }
         }
@@ -1786,6 +2005,12 @@ bool FgNhgOrch::doTaskFgNhgMember(const KeyOpFieldsValuesTuple & t)
             SWSS_LOG_INFO("FG_NHG entry not received yet, continue");
             return false;
         }
+        // Flag error if FGNHG was created with match_mode as prefix_based and member is added
+        else if (fgNhg_entry->second.match_mode == FGMatchMode::PREFIX_BASED)
+        {
+            SWSS_LOG_ERROR("Received FG_NHG member for prefix-based match_mode, not a supported operation");
+            return true;
+        }
         else
         {
             /* skip addition if next-hop already exists */
@@ -1894,9 +2119,9 @@ bool FgNhgOrch::doTaskFgNhgMember(const KeyOpFieldsValuesTuple & t)
     }
     return true;
 }
-        
 
-void FgNhgOrch::doTask(Consumer& consumer) 
+
+void FgNhgOrch::doTask(Consumer& consumer)
 {
     SWSS_LOG_ENTER();
     const string & table_name = consumer.getTableName();
diff --git a/orchagent/fgnhgorch.h b/orchagent/fgnhgorch.h
index b7ec844a590..1c39a4e5b1f 100644
--- a/orchagent/fgnhgorch.h
+++ b/orchagent/fgnhgorch.h
@@ -15,9 +15,10 @@
 #include <map>
 
 typedef uint32_t Bank;
+typedef uint32_t HashBucketIdx;
 typedef std::set<NextHopKey> ActiveNextHops;
 typedef std::vector<sai_object_id_t> FGNextHopGroupMembers;
-typedef std::vector<uint32_t> HashBuckets;
+typedef std::vector<HashBucketIdx> HashBuckets;
 typedef std::map<NextHopKey, HashBuckets> FGNextHopGroupMap;
 typedef std::vector<FGNextHopGroupMap> BankFGNextHopGroupMap;
 typedef std::map<Bank,Bank> InactiveBankMapsToBank;
@@ -56,8 +57,10 @@ typedef std::unordered_map<string, std::vector<IpAddress>> Links;
 enum FGMatchMode
 {
     ROUTE_BASED,
-    NEXTHOP_BASED
+    NEXTHOP_BASED,
+    PREFIX_BASED
 };
+
 /* Store the indices occupied by a bank */
 typedef struct
 {
@@ -70,6 +73,7 @@ typedef struct FgNhgEntry
     string fg_nhg_name;                               // Name of FG NHG group configured by user
     uint32_t configured_bucket_size;                  // Bucket size configured by user
     uint32_t real_bucket_size;                        // Real bucket size as queried from SAI
+    uint32_t max_next_hops;                           // For match_mode==prefix-based. Maximum number of next hops in the FG NHG
     NextHops next_hops;                               // The IP to Bank mapping configured by user
     Links links;                                      // Link to IP map for oper changes
     std::vector<IpPrefix> prefixes;                   // Prefix which desires FG behavior
@@ -78,9 +82,9 @@ typedef struct FgNhgEntry
 } FgNhgEntry;
 
 /* Map from IP prefix to user configured FG NHG entries */
-typedef std::map<IpPrefix, FgNhgEntry*> FgNhgPrefixes; 
+typedef std::map<IpPrefix, FgNhgEntry*> FgNhgPrefixes;
 /* Map from IP address to user configured FG NHG entries */
-typedef std::map<IpAddress, FgNhgEntry*> FgNhgMembers; 
+typedef std::map<IpAddress, FgNhgEntry*> FgNhgMembers;
 /* Main structure to hold user configuration */
 typedef std::map<FgNhg, FgNhgEntry> FgNhgs;
 
@@ -133,13 +137,18 @@ class FgNhgOrch : public Orch, public Observer
     WarmBootRecoveryMap m_recoveryMap;
 
     bool setNewNhgMembers(FGNextHopGroupEntry &syncd_fg_route_entry, FgNhgEntry *fgNhgEntry,
-                    std::vector<BankMemberChanges> &bank_member_changes, 
+                    std::vector<BankMemberChanges> &bank_member_changes,
                     std::map<NextHopKey,sai_object_id_t> &nhopgroup_members_set, const IpPrefix&);
+    bool sprayBankNhgMembers(FGNextHopGroupEntry &syncd_fg_route_entry, const IpPrefix &ipPrefix,
+                    BankIndexRange hash_idx_range, FgNhgEntry *fgNhgEntry,
+                    uint32_t bank, BankMemberChanges &bank_member_change,
+                    std::map<NextHopKey,sai_object_id_t> &nhopgroup_members_set);
+
     bool computeAndSetHashBucketChanges(FGNextHopGroupEntry *syncd_fg_route_entry,
                     FgNhgEntry *fgNhgEntry, std::vector<BankMemberChanges> &bank_member_changes,
                     std::map<NextHopKey,sai_object_id_t> &nhopgroup_members_set, const IpPrefix&);
     bool setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_route_entry, FgNhgEntry *fgNhgEntry,
-                    uint32_t bank, uint32_t syncd_bank, std::vector<BankMemberChanges> bank_member_changes,
+                    uint32_t syncd_bank, BankMemberChanges bank_member_changes,
                     std::map<NextHopKey,sai_object_id_t> &nhopgroup_members_set, const IpPrefix&);
     bool setInactiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_route_entry, FgNhgEntry *fgNhgEntry,
                     uint32_t bank,std::vector<BankMemberChanges> &bank_member_changes,
diff --git a/orchagent/orchdaemon.cpp b/orchagent/orchdaemon.cpp
index 1a63f1fe8ce..119a45e3591 100644
--- a/orchagent/orchdaemon.cpp
+++ b/orchagent/orchdaemon.cpp
@@ -496,7 +496,7 @@ bool OrchDaemon::init()
      * when iterating ConsumerMap. This is ensured implicitly by the order of keys in ordered map.
      * For cases when Orch has to process tables in specific order, like PortsOrch during warm start, it has to override Orch::doTask()
      */
-    m_orchList = { gSwitchOrch, gCrmOrch, gPortsOrch, gBufferOrch, gFlowCounterRouteOrch, gIntfsOrch, gNeighOrch, gNhgMapOrch, gNhgOrch, gCbfNhgOrch, gRouteOrch, gCoppOrch, gQosOrch, wm_orch, gPolicerOrch, gTunneldecapOrch, sflow_orch, gDebugCounterOrch, gMacsecOrch, bgp_global_state_orch, gBfdOrch, gSrv6Orch, gMuxOrch, mux_cb_orch, gMonitorOrch, gStpOrch};
+    m_orchList = { gSwitchOrch, gCrmOrch, gPortsOrch, gBufferOrch, gFlowCounterRouteOrch, gIntfsOrch, gNeighOrch, gNhgMapOrch, gNhgOrch, gCbfNhgOrch, gFgNhgOrch, gRouteOrch, gCoppOrch, gQosOrch, wm_orch, gPolicerOrch, gTunneldecapOrch, sflow_orch, gDebugCounterOrch, gMacsecOrch, bgp_global_state_orch, gBfdOrch, gSrv6Orch, gMuxOrch, mux_cb_orch, gMonitorOrch, gStpOrch};
 
     bool initialize_dtel = false;
     if (platform == BFN_PLATFORM_SUBSTRING || platform == VS_PLATFORM_SUBSTRING)
@@ -593,7 +593,6 @@ bool OrchDaemon::init()
     m_orchList.push_back(gNatOrch);
     m_orchList.push_back(gMlagOrch);
     m_orchList.push_back(gIsoGrpOrch);
-    m_orchList.push_back(gFgNhgOrch);
     m_orchList.push_back(mux_st_orch);
     m_orchList.push_back(nvgre_tunnel_orch);
     m_orchList.push_back(nvgre_tunnel_map_orch);
diff --git a/tests/test_fgnhg.py b/tests/test_fgnhg.py
index 645853e24cf..3cb7e8f1fca 100644
--- a/tests/test_fgnhg.py
+++ b/tests/test_fgnhg.py
@@ -3,6 +3,7 @@
 import time
 import json
 import pytest
+import random
 
 from dvslib.dvs_common import wait_for_result
 from swsscommon import swsscommon
@@ -148,7 +149,7 @@ def get_nh_oid_map(asic_db):
     assert nh_oid_map != {}
     return nh_oid_map
 
-def verify_programmed_fg_asic_db_entry(asic_db,nh_memb_exp_count,nh_oid_map,nhgid,bucket_size):
+def verify_programmed_fg_asic_db_entry(asic_db,prev_memb_dict,num_exp_changes,nh_memb_exp_count,nh_oid_map,nhgid,bucket_size):
     def _access_function():
         false_ret = (False, None)
         ret = True
@@ -182,7 +183,8 @@ def _access_function():
                nh_oid == "0" or
                nh_oid_map.get(nh_oid,"NULL") == "NULL" or
                nh_oid_map.get(nh_oid) not in nh_memb_exp_count):
-                print("Invalid nh: nh_oid " + nh_oid + " index " + str(index))
+                print("Invalid nh: nh_oid " + nh_oid + " index " + str(index) +
+                      " member: " +  member)
                 if nh_oid_map.get(nh_oid,"NULL") == "NULL":
                     print("nh_oid is null")
                 if nh_oid_map.get(nh_oid) not in nh_memb_exp_count:
@@ -190,22 +192,24 @@ def _access_function():
                 return false_ret
             memb_dict[index] = nh_oid_map.get(nh_oid)
         idxs = [0]*bucket_size
+        num_changes = 0
         for idx,memb in memb_dict.items():
             nh_memb_count[memb] = 1 + nh_memb_count[memb]
             idxs[idx] = idxs[idx] + 1
-
+            if memb != prev_memb_dict.get(idx, "NULL"):
+                num_changes = num_changes + 1
+                #print("Change detected at index " + str(idx) + " old nh " + prev_memb_dict.get(idx, "NULL") + " new nh " + memb)
         for key in nh_memb_exp_count:
             ret = ret and (nh_memb_count[key] == nh_memb_exp_count[key])
         for idx in idxs:
             ret = ret and (idx == 1)
-        if ret != True:
-            print("Expected member count was " + str(nh_memb_exp_count) + " Received was " + str(nh_memb_count))
-            print("Indexes arr was " + str(idxs))
-        return (ret, nh_memb_count)
+        if num_changes != num_exp_changes:
+            ret = False
+        return ret, memb_dict
 
-    status, result = wait_for_result(_access_function)
-    assert status, f"Exact match not found: expected={nh_memb_exp_count}, received={result}"
-    return result
+    status, new_memb_dict = wait_for_result(_access_function)
+    assert status, f"Exact match not found: expected={nh_memb_exp_count}, received={nh_memb_count}"
+    return new_memb_dict
 
 def shutdown_link(dvs, db, port):
     dvs.servers[port].runcmd("ip link set down dev eth0") == 0
@@ -242,17 +246,52 @@ def verify_programmed_fg_state_db_entry(state_db, fg_nhg_prefix, nh_memb_exp_cou
     for idx,memb in memb_dict.items():
         assert memb == 0
 
-def validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+def verify_fg_state_db_for_even_distribution(state_db, fg_nhg_prefix, bucket_size, nh_ip_count):
+    def _access_function():
+        false_ret = (False, '')
+        ret = True
+        keys = state_db.get_keys("FG_ROUTE_TABLE")
+        if not keys:
+            return false_ret
+        for key in keys:
+            if key != fg_nhg_prefix:
+                continue
+            fvs = state_db.get_entry("FG_ROUTE_TABLE", key)
+            if not fvs:
+                return false_ret
+            member_count = {}
+            for key, value in fvs.items():
+                    if value not in member_count:
+                        member_count[value] = 0
+                    member_count[value] = member_count[value] + 1
+
+        # Verify that values in the member_count dictionary don't differ by more than 1
+        if member_count:
+            min_count = min(member_count.values())
+            max_count = max(member_count.values())
+            if (min_count == bucket_size//nh_ip_count) and (max_count - min_count <= 1) and (sum(member_count.values()) == bucket_size):
+                ret = True
+            else:
+                ret = False
+
+        else:
+            ret = False
+        return ret, member_count
+
+    status, member_count = wait_for_result(_access_function)
+    assert status, f"Member count distribution is uneven"
+
+def validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, prev_memb_dict, num_exp_changes,
                                 fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size):
     state_db_entry_memb_exp_count = {}
 
     for ip, cnt in nh_memb_exp_count.items():
         state_db_entry_memb_exp_count[ip + '@' + ip_to_if_map[ip]] = cnt
-
-    verify_programmed_fg_asic_db_entry(asic_db,nh_memb_exp_count,nh_oid_map,nhgid,bucket_size)
+    next_memb_dict = verify_programmed_fg_asic_db_entry(asic_db,prev_memb_dict,num_exp_changes,nh_memb_exp_count,nh_oid_map,nhgid,bucket_size)
     verify_programmed_fg_state_db_entry(state_db, fg_nhg_prefix, state_db_entry_memb_exp_count)
+    return next_memb_dict
 
-def program_route_and_validate_fine_grained_ecmp(app_db, asic_db, state_db, ip_to_if_map,
+def program_route_and_validate_fine_grained_ecmp(app_db, asic_db, state_db, ip_to_if_map, prev_memb_dict, num_exp_changes,
                             fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size):
     ips = ""
     ifs = ""
@@ -267,8 +306,25 @@ def program_route_and_validate_fine_grained_ecmp(app_db, asic_db, state_db, ip_t
     ps = swsscommon.ProducerStateTable(app_db, ROUTE_TB)
     fvs = swsscommon.FieldValuePairs([("nexthop", ips), ("ifname", ifs)])
     ps.set(fg_nhg_prefix, fvs)
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                          fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    new_memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        prev_memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    return new_memb_dict
+
+def program_route_and_validate_distribtution(app_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_ips, bucket_size):
+    ips = ""
+    ifs = ""
+    for ip in nh_ips:
+        if ips == "":
+            ips = ip
+            ifs = ip_to_if_map[ip]
+        else:
+            ips = ips + "," + ip
+            ifs = ifs + "," + ip_to_if_map[ip]
+
+    ps = swsscommon.ProducerStateTable(app_db, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop", ips), ("ifname", ifs)])
+    ps.set(fg_nhg_prefix, fvs)
+    verify_fg_state_db_for_even_distribution(state_db, fg_nhg_prefix, bucket_size, len(nh_ips))
 
 def create_interface_n_fg_ecmp_config(dvs, nh_range_start, nh_range_end, fg_nhg_name):
     ip_to_if_map = {}
@@ -317,6 +373,12 @@ def fine_grained_ecmp_base_test(dvs, match_mode):
     bucket_size = 60
     ip_to_if_map = {}
 
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
     fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode}
     create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
 
@@ -382,12 +444,14 @@ def fine_grained_ecmp_base_test(dvs, match_mode):
     nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
 
     nh_oid_map = get_nh_oid_map(asic_db)
+    memb_dict = {}
 
     ### Test scenarios with bank 0 having 0 members up and only bank 1 having members
     # ARP is not resolved for 10.0.0.7, so fg nhg should be created without 10.0.0.7
     nh_memb_exp_count = {"10.0.0.9":30,"10.0.0.11":30}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Resolve ARP for 10.0.0.7
     asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
@@ -397,8 +461,9 @@ def fine_grained_ecmp_base_test(dvs, match_mode):
 
     # Now that ARP was resolved, 10.0.0.7 should be added as a valid fg nhg member
     nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 20
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Test warm reboot
     run_warm_reboot(dvs)
@@ -406,104 +471,123 @@ def fine_grained_ecmp_base_test(dvs, match_mode):
     nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
     nh_oid_map = get_nh_oid_map(asic_db)
     nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Bring down 1 next hop in bank 1
     nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
-    program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                        fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Bring down 2 next hop and bring up 1 next hop in bank 1
     nh_memb_exp_count = {"10.0.0.9":60}
-    program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                        fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 60
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Bring up 1 next hop in bank 1
     nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
-    program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                        fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Bring up some next-hops in bank 0 for the 1st time
     nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
-    program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                        fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Test warm reboot
     run_warm_reboot(dvs)
     asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
     nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
     nh_oid_map = get_nh_oid_map(asic_db)
+    num_exp_changes = 0
     nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Bring down 1 next-hop from bank 0, and 2 next-hops from bank 1
     nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.11":30}
-    program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                        fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Bring down 1 member and bring up 1 member in bank 0 at the same time
     nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.3":15,"10.0.0.11":30}
-    program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                        fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 15
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Bring down 2 members and bring up 1 member in bank 0 at the same time
     nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30}
-    program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                        fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Bring up 2 members and bring down 1 member in bank 0 at the same time
     nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.3":15,"10.0.0.11":30}
-    program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                        fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Bringup arbitrary # of next-hops from both banks at the same time
     nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
-    program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                        fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Bring all next-hops in bank 1 down
     nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.5":20}
-    program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                        fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Make next-hop changes to bank 0 members, given bank 1 is still down
     nh_memb_exp_count = {"10.0.0.1":30,"10.0.0.5":30}
-    program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                        fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Bringup 1 member in bank 1 again
     nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.11":30}
-    program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                        fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Test 2nd,3rd memb up in bank
     nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
-    program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                        fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # bring all links down one by one
     shutdown_link(dvs, app_db, 0)
     shutdown_link(dvs, app_db, 1)
     nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 15
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     shutdown_link(dvs, app_db, 2)
+    num_exp_changes = 30
     nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     shutdown_link(dvs, app_db, 3)
     nh_memb_exp_count = {"10.0.0.9":30,"10.0.0.11":30}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 20
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     shutdown_link(dvs, app_db, 4)
+    num_exp_changes = 30
     nh_memb_exp_count = {"10.0.0.11":60}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Bring down last link, there shouldn't be a crash or other bad orchagent state because of this
     shutdown_link(dvs, app_db, 5)
@@ -516,9 +600,10 @@ def fine_grained_ecmp_base_test(dvs, match_mode):
     asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
     nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
     nh_oid_map = get_nh_oid_map(asic_db)
+    num_exp_changes = 60
     nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.9":30}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     startup_link(dvs, app_db, 5)
     # Perform a route table update, Update the route to contain 10.0.0.3 as well, since Ethernet4 associated with it
@@ -530,31 +615,36 @@ def fine_grained_ecmp_base_test(dvs, match_mode):
     # 10.0.0.11 associated with newly brought up link 5 should be updated in FG ecmp
     # 10.0.0.3 addition per above route table change should have no effect
     nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 20
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     startup_link(dvs, app_db, 2)
     nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 30
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     startup_link(dvs, app_db, 0)
     nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 15
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # remove fgnhg member
     remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0.1")
     nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 15
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # add fgnhg member
     fvs = {"FG_NHG": fg_nhg_name, "bank": "0"}
     create_entry(config_db, FG_NHG_MEMBER, "10.0.0.1", fvs)
     nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 15
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Remove route
     asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
@@ -589,10 +679,12 @@ def fine_grained_ecmp_base_test(dvs, match_mode):
 
         nh_oid_map = {}
         nh_oid_map = get_nh_oid_map(asic_db)
+        memb_dict = {}
 
         nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
-        validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                                fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 60
+        memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                                memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # remove fgnhg prefix: The fine grained route should transition to regular ECMP/route
         remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
@@ -624,6 +716,624 @@ def fine_grained_ecmp_base_test(dvs, match_mode):
         remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
+def fine_grained_ecmp_match_mode_prefix_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 6
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 60
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for 3 next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    # Add route with 3 next-hops
+    print("Add route with 3 next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members
+    print("Test scenarios with 3 members")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                       memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 next hop
+    print("FGNHG Bring down 1 next hop")
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # (Bring down 2 next hop and bring up 1 next hop)
+    print("Bring down 2 next hop and bring up 1 next hop")
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 60
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 next hops
+    print("Bring up 2 next hops")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 3 more next-hops
+    print("Bring up 3 more next-hops")
+    # First Resolve ARP for 3 more next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:08")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:09")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    # Test warm reboot
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 3 next-hops
+    print("Bring down 3 next-hops")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.5":20,"10.0.0.11":20}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 member and bring up 1 member at the same time
+    print("Bring down 1 member and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 2 members and bring up 1 member at the same time
+    print("Bring down 2 members and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 members and bring down 1 member at the same time
+    print("Bring up 2 members and bring down 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bringup all the inactive nexthops at the same time
+    print("Bringup all the inactive nexthops at the same time")
+    num_exp_changes = 30
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+     # Remove route
+    print("Remove route")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+
+    # validate routes and nhg member in asic db, route entry in state db are removed
+    print("validate routes and nhg member in asic db, route entry in state db are removed")
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    # Remove fgnhg prefix and group
+    print("Remove fgnhg prefix")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+    # Nothing we can wait for in terms of db entries, we sleep here
+    # to give the sw enough time to delete the entry
+    time.sleep(1)
+
+    # Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see
+    # standard(non-Fine grained) ECMP behavior
+    print("Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see")
+    print("standard(non-Fine grained) ECMP behavior")
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+            ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+    validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3)
+
+    # add back fgnhg group and prefix: The regular route should transition to fine grained ECMP
+    print("add fgnhg group and prefix: The regular route should transition to fine grained ECMP")
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    # Validate the transistion to Fine Grained ECMP
+    print("Validate the transistion to Fine Grained ECMP")
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = {}
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # remove prefix entry
+    print("remove prefix entry")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+
+    # Cleanup all FG, arp and interface
+    print("Cleanup all FG, arp and interface")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, IF_TB, ip_pref_key)
+        remove_entry(config_db, IF_TB, if_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_multi_route_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM0_NHs = 6
+    NUM1_NHs = 4
+    fg_nhg_name0 = "fgnhg0_v4"
+    fg_nhg_name1 = "fgnhg1_v4"
+    fg_nhg_prefix0 = "2.2.2.0/24"
+    fg_nhg_prefix1 = "3.3.3.0/24"
+    bucket0_size = 60
+    bucket1_size = 24
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    ### Create first fine grained next hop group and prefix
+    fvs = {"bucket_size": str(bucket0_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM0_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name0, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name0}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix0, fvs)
+
+    ### Create shared interfaces (Last two Nexthops for the 1st prefix are shared with the 2nd prefix)
+    for i in range(0,NUM0_NHs+NUM1_NHs-2):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        fvs = {"vlanid": str((i+1)*4)}
+        create_entry(config_db, VLAN_TB, vlan_name_key, fvs)
+        fvs = {"tagging_mode": "untagged"}
+        create_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key, fvs)
+        create_entry(config_db, VLAN_IF_TB, vlan_name_key, fvs_nul)
+        create_entry(config_db, VLAN_IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = vlan_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    ### Create Route for the first Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan16,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix0, fvs)
+    # No ASIC_DB entry we can wait for since ARP is not resolved yet,
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix0:
+            found_route = True
+            break
+
+    # Since we didn't populate ARP yet, route should point to RIF for kernel arp resolution to occur
+    assert (found_route == True)
+    validate_asic_nhg_router_interface(asic_db, fg_nhg_prefix0)
+
+    # Add ARP entries for both prefixes
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03")
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+    dvs.runcmd("arp -s 10.0.0.13 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.15 00:00:00:00:00:08")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 8)
+
+    # verify correct number of nhg members were created for the first prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size)
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Create 2nd next hop group and prefix
+    fvs = {"bucket_size": str(bucket1_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM1_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name1, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name1}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix1, fvs)
+
+    # Create Route for the 2nd Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.13,10.0.0.15,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan28,Vlan32,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix1, fvs)
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix1:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    # verify correct number of nhg members were created for the 2nd prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members for the first prefix and 4 members for the 2nd prefix
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict0 = {}
+    num_exp_changes = 60
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    memb_dict1 = {}
+    num_exp_changes = 24
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Test warm reboot
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    num_exp_changes = 0
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 1 common next hop
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":8,"10.0.0.15":8,"10.0.0.11":8}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring it back up
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 3 next-hops
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 40
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":12,"10.0.0.11":12}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bringup all the inactive nexthops at the same time
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 50
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # remove prefix entries
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix0)
+    ps._del(fg_nhg_prefix0)
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix1)
+    ps._del(fg_nhg_prefix1)
+
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix0)
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix1)
+    # Cleanup all FG, arp and interface
+    remove_entry(config_db, "FG_NHG", fg_nhg_name0)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name1)
+
+    for i in range(0,NUM0_NHs+NUM1_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, VLAN_IF_TB, ip_pref_key)
+        remove_entry(config_db, VLAN_IF_TB, vlan_name_key)
+        remove_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key)
+        remove_entry(config_db, VLAN_TB, vlan_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_even_distribution_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 16
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 256
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for all NUM_NHs next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    for i in range(NUM_NHs):
+        dvs.runcmd(f"arp -s 10.0.0.{1 + i * 2} 00:00:00:00:00:{1 + i * 2:02x}")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + NUM_NHs)
+
+    # Add route with NUM_NHs next-hops
+    print(f"Add route with {NUM_NHs} next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop", ",".join([f"10.0.0.{1 + i * 2}" for i in range(NUM_NHs)])),
+        ("ifname", ",".join([f"Ethernet{i * 4}" for i in range(NUM_NHs)]))])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    ### Start with NUM_NHs members and verify that distribution is even after a series of add/remove operations
+    for _ in range(50):
+        print()
+        print(f"Iteration: {_}, started test with {NUM_NHs} members, bucket_size: {bucket_size}")
+        nh_ips = [f"10.0.0.{1 + i * 2}" for i in range(NUM_NHs)]
+        program_route_and_validate_distribtution(app_db.db_connection, state_db, ip_to_if_map,
+                                                         fg_nhg_prefix, nh_ips, bucket_size)
+
+        ### remove 1 to 3 nexthops in each step and verify that distribution is even
+        removed_ips = []
+        while len(nh_ips) > 1:
+            num_to_remove = random.randint(1, min(3, len(nh_ips) - 1))
+            removed_in_iteration = []
+            for _ in range(num_to_remove):
+                removed_ip = nh_ips.pop()
+                removed_ips.append(removed_ip)
+                removed_in_iteration.append(removed_ip)
+            print(f"Removed IPs: {removed_in_iteration}, No. of remaining IPs: {len(nh_ips)}")
+            program_route_and_validate_distribtution(app_db.db_connection, state_db, ip_to_if_map,
+                                                         fg_nhg_prefix, nh_ips, bucket_size)
+
+        ### add 1-3 nexthops in each step and verify that distribution stays even
+        while len(nh_ips) < NUM_NHs:
+            num_to_add = random.randint(1, min(3, len(removed_ips)))
+            added_in_iteration = []
+            for _ in range(num_to_add):
+                if removed_ips:
+                    added_ip = removed_ips.pop(0)
+                    nh_ips.append(added_ip)
+                    added_in_iteration.append(added_ip)
+            print(f"Added IPs: {added_in_iteration}, Total IPs: {len(nh_ips)}")
+            program_route_and_validate_distribtution(app_db.db_connection, state_db, ip_to_if_map,
+                                                     fg_nhg_prefix, nh_ips, bucket_size)
+
+    # Remove route
+    print("Remove route")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+
+    # validate routes and nhg member in asic db, route entry in state db are removed
+    print("validate routes and nhg member in asic db, route entry in state db are removed")
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+
+    # Cleanup all FG, arp and interface
+    print("Cleanup all FG, arp and interface")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, IF_TB, ip_pref_key)
+        remove_entry(config_db, IF_TB, if_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+
 class TestFineGrainedNextHopGroup(object):
     def test_fgnhg_matchmode_route(self, dvs, testlog):
         '''
@@ -637,6 +1347,24 @@ def test_fgnhg_matchmode_nexthop(self, dvs, testlog):
         '''
         fine_grained_ecmp_base_test(dvs, 'nexthop-based')
 
+    def test_fgnhg_matchmode_prefix(self, dvs, testlog):
+        '''
+        Test for match_mode prefix-based
+        '''
+        fine_grained_ecmp_match_mode_prefix_test(dvs)
+
+    def test_fgnhg_matchmode_prefix_multi_route(self, dvs, testlog):
+        '''
+        Test for match_mode prefix-based with multiple routes
+        '''
+        fine_grained_ecmp_match_mode_prefix_multi_route_test(dvs);
+
+    def test_fgnhg_matchmode_prefix_even_distribution(self, dvs, testlog):
+        '''
+        Test for match_mode prefix-based with up to 16 nexthops and even distribution
+        '''
+        fine_grained_ecmp_match_mode_prefix_even_distribution_test(dvs);
+
     def test_fgnhg_more_nhs_nondiv_bucket_size(self, dvs, testlog):
         '''
         Test Fine Grained ECMP with a greater number of FG members and
@@ -680,49 +1408,58 @@ def test_fgnhg_more_nhs_nondiv_bucket_size(self, dvs, testlog):
         nh_oid_map = get_nh_oid_map(asic_db)
 
         # The route had been created with 0 members in bank
+        memb_dict = {}
         nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.11":64}
-        validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                                fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128
+        memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Add 2 nhs to both bank 0 and bank 1
         nh_memb_exp_count = {"10.0.0.1":22,"10.0.0.3":21,"10.0.0.5":21,"10.0.0.11":22,
                 "10.0.0.13":21,"10.0.0.15":21}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-22-22
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Add 2 more nhs to both bank 0 and bank 1
         nh_memb_exp_count = {"10.0.0.1":13,"10.0.0.3":13,"10.0.0.5":13,"10.0.0.7":12,
                 "10.0.0.9":13,"10.0.0.11":13,"10.0.0.13":13,"10.0.0.15":13,"10.0.0.17":12,"10.0.0.19":13}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-6*13
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Remove 1 nh from bank 0 and remove 2 nhs from bank 1
         nh_memb_exp_count = {"10.0.0.3":16,"10.0.0.5":16,"10.0.0.7":16,"10.0.0.9":16,
                 "10.0.0.11":22,"10.0.0.13":21,"10.0.0.19":21}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-13-13-12-13-13-13-13
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Remove 1 nh from bank 0 and add 1 nh to bank 1
         nh_memb_exp_count = {"10.0.0.3":22,"10.0.0.7":21,"10.0.0.9":21,"10.0.0.13":16,
                 "10.0.0.15":16,"10.0.0.17":16,"10.0.0.19":16}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-5*16
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Remove 2 nh from bank 0 and remove 3 nh from bank 1
         nh_memb_exp_count = {"10.0.0.7":64,"10.0.0.11":64}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-21
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Add 2 nhs to bank 0 and remove all nh from bank 1
         nh_memb_exp_count = {"10.0.0.5":42,"10.0.0.7":44,"10.0.0.9":42}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-22
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Add 2 nhs to bank 0 and add 1 nh to bank 1
         nh_memb_exp_count = {"10.0.0.1":12,"10.0.0.3":13,"10.0.0.5":13,"10.0.0.7":13,
                 "10.0.0.9":13,"10.0.0.11":64}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-13-13-13
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Remove route
         # remove prefix entry
@@ -739,7 +1476,7 @@ def test_fgnhg_more_nhs_nondiv_bucket_size(self, dvs, testlog):
     def test_fgnhg_matchmode_nexthop_multi_route(self, dvs, testlog):
         '''
         Test route/nh transitions to/from Fine Grained ECMP and Regular ECMP.
-        Create multiple prefixes pointing to the Fine Grained nhs and ensure 
+        Create multiple prefixes pointing to the Fine Grained nhs and ensure
         fine grained ECMP ASIC objects were created for this scenario as expected.
         '''
         app_db = dvs.get_app_db()
@@ -788,11 +1525,13 @@ def test_fgnhg_matchmode_nexthop_multi_route(self, dvs, testlog):
         nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
 
         nh_oid_map = get_nh_oid_map(asic_db)
+        memb_dict = {}
 
         # The route had been created with 0 members in bank
         nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.5":64}
-        validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                                fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128
+        memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Add a 2nd prefix associated with the same set of next-hops
         fg_nhg_prefix_2 = "5.5.5.0/16"
@@ -802,8 +1541,9 @@ def test_fgnhg_matchmode_nexthop_multi_route(self, dvs, testlog):
         asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size*2)
         nhgid_2 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix_2, bucket_size)
         nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.5":64}
-        validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                                fg_nhg_prefix_2, nh_memb_exp_count, nh_oid_map, nhgid_2, bucket_size)
+        num_exp_changes = 0
+        memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix_2, nh_memb_exp_count, nh_oid_map, nhgid_2, bucket_size)
 
         # Add a 3rd prefix with a next-hop(10.0.0.9) not defined for FG ECMP
         # Should end up as regular ECMP
@@ -820,8 +1560,9 @@ def test_fgnhg_matchmode_nexthop_multi_route(self, dvs, testlog):
         nhgid_3 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix_3, bucket_size)
         asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size*3)
         nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.5":64}
-        validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                                fg_nhg_prefix_3, nh_memb_exp_count, nh_oid_map, nhgid_3, bucket_size)
+        num_exp_changes = 0
+        memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix_3, nh_memb_exp_count, nh_oid_map, nhgid_3, bucket_size)
         # Add the 10.0.0.9 next-hop again, it should transition back to regular ECMP
         fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.5,10.0.0.9"),
                 ("ifname", "Ethernet0,Ethernet8,Ethernet16")])
@@ -836,11 +1577,13 @@ def test_fgnhg_matchmode_nexthop_multi_route(self, dvs, testlog):
 
         # Change FG nhs for one route, ensure that the other route nh is unaffected
         nh_memb_exp_count = {"10.0.0.1":32,"10.0.0.3":32,"10.0.0.5":32,"10.0.0.7":32}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 64
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
         nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.5":64}
-        validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                                fg_nhg_prefix_2, nh_memb_exp_count, nh_oid_map, nhgid_2, bucket_size)
+        num_exp_changes = 64
+        memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix_2, nh_memb_exp_count, nh_oid_map, nhgid_2, bucket_size)
 
         # Remove route
         # remove prefix entry
@@ -850,8 +1593,9 @@ def test_fgnhg_matchmode_nexthop_multi_route(self, dvs, testlog):
         asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
         # Ensure that 2nd route is still here and then delete it
         nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.5":64}
-        validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                                fg_nhg_prefix_2, nh_memb_exp_count, nh_oid_map, nhgid_2, bucket_size)
+        num_exp_changes = 0
+        memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix_2, nh_memb_exp_count, nh_oid_map, nhgid_2, bucket_size)
         # Delete the 2nd route as well
         asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix_2)
         ps._del(fg_nhg_prefix_2)


Reviews:
user: login: anish-n
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-16T21:38:09Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-02-19T06:14:37Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-02-19T06:16:13Z
user: login: manamand2020
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-15T00:57:35Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-26T22:48:47Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T00:24:13Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T01:17:51Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T01:19:04Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T01:21:13Z
user: login: anish-n
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-05T20:27:08Z
user: login: anish-n
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-05T20:30:14Z
user: login: anish-n
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-06T20:01:33Z
user: login: anish-n
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-06T21:55:03Z
user: login: anish-n
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-06T22:44:05Z
user: login: anish-n
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-06T23:30:16Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-11T18:11:19Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-11T18:17:31Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-11T18:17:37Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-11T18:17:45Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-11T18:18:01Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-11T18:18:12Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-11T18:18:58Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-11T18:19:08Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-11T18:19:12Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-15T17:53:00Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-15T17:54:21Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-17T15:06:24Z
user: login: anish-n
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-26T19:45:52Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T17:09:42Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T17:09:55Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T17:10:01Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T17:10:08Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T17:10:15Z
user: login: ashutosh-agrawal
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-28T17:16:31Z
user: login: anish-n
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T17:16:53Z
user: login: prsunny
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-29T17:04:09Z
user: login: github-advanced-security[bot]
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-29T17:06:40Z

Review comments:
diff_hunk: @@ -15,9 +15,10 @@
 #include <map>
 
 typedef uint32_t Bank;
+typedef uint32_t HashBucketIdx;
 typedef std::set<NextHopKey> ActiveNextHops;
-typedef std::vector<sai_object_id_t> FGNextHopGroupMembers;
-typedef std::vector<uint32_t> HashBuckets;
+typedef std::map<HashBucketIdx, sai_object_id_t> FGNextHopGroupMembers;
path: orchagent/fgnhgorch.h
user: login: anish-n
body: why does this need to be a map? The vector based implementation is practically already a map already because the hash bucket index is the position in the vector with the value being the sai object.
created_at: 2025-02-13T20:05:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -610,22 +651,72 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        bool ends_with_balance = num_nhs_with_one_more ? false : true;
+        auto active_nh_list = bank_member_change.active_nhs;
+        size_t act_nh_exp_size = exp_bucket_size;
+
+        if (!ends_with_balance)
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: The use of double negative and multiple if else statements, with ternary defining the ends_with_balance  makes this logic complicated, lets simplify this by directly using an if statement based on num_nhs_with_one_more. 

This can be simplified to ```if (num_nhs_with_one_more > 0)```, the logic is currently doing:
```
bool ends_with_balance = num_nhs_with_one_more ? false : true;

if (!ends_with_balance) { act_nh_exp_size++; } 
```
created_at: 2025-02-15T16:20:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -152,81 +152,112 @@ bool FgNhgOrch::bake()
 void FgNhgOrch::calculateBankHashBucketStartIndices(FgNhgEntry *fgNhgEntry)
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: This entire file has a lot of variable name changes which results in a huge PR with changes touching many different parts, can we please separate the variable nit changes over the actual changes required for this PR. 
created_at: 2025-02-15T16:27:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -56,25 +57,34 @@ typedef std::unordered_map<string, std::vector<IpAddress>> Links;
 enum FGMatchMode
 {
     ROUTE_BASED,
-    NEXTHOP_BASED
+    NEXTHOP_BASED,
+    DYNAMIC_ROUTE_BASED
 };
+enum FGNexthopMode
+{
+    STATIC_FGNHG,
+    DYNAMIC_FGNHG
+};
+
 /* Store the indices occupied by a bank */
 typedef struct
 {
     uint32_t start_index;
     uint32_t end_index;
-} BankIndexRange;
+} HashIndexRange;
path: orchagent/fgnhgorch.h
user: login: anish-n
body: don't quite agree with this renaming. This is a per bank hash bucket index range, if there is just 1 bank, bankindexrange = hashindexrange, otherwise its a bankindexrange. I think its more intuitive if this is named bankindexrange. 
created_at: 2025-02-15T16:31:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -566,19 +605,21 @@ bool FgNhgOrch::invalidNextHopInNextHopGroup(const NextHopKey& nexthop)
  * Thereby achieving consistent and layered hashing.
  */
 bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_route_entry, FgNhgEntry *fgNhgEntry,
-        uint32_t bank, uint32_t syncd_bank, std::vector<BankMemberChanges> bank_member_changes, 
+        uint32_t syncd_bank, BankMemberChanges bank_member_change,
         std::map<NextHopKey,sai_object_id_t> &nhopgroup_members_set, const IpPrefix &ipPrefix)
 {
     SWSS_LOG_ENTER();
 
-    BankMemberChanges bank_member_change = bank_member_changes[bank];
     uint32_t add_idx = 0, del_idx = 0;
-    FGNextHopGroupMap *bank_fgnhg_map = &(syncd_fg_route_entry->syncd_fgnhg_map[syncd_bank]);
+    FGNextHopGroupMap *fgnhg_bucket_map = &(syncd_fg_route_entry->syncd_fgnhg_map[syncd_bank]);
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: I think that bank_fgnhg_map is more intuitive, this is the bucket map for the particular bank and not for a fgnhg. Please can you change this back to bank_fgnhg_map?
created_at: 2025-02-15T17:31:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -610,22 +651,72 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        bool ends_with_balance = num_nhs_with_one_more ? false : true;
+        auto active_nh_list = bank_member_change.active_nhs;
+        size_t act_nh_exp_size = exp_bucket_size;
+
+        if (!ends_with_balance)
+        {
+            act_nh_exp_size++;
+        }
 
+        // arrange the active nhs in increasing order of size of their hash bucket map
+        // so that we add buckets to the nhs which have lower bucket size first
+        auto compare_active_buckets = [&](NextHopKey e1, NextHopKey e2) {
+            return fgnhg_bucket_map->at(e1).size() < fgnhg_bucket_map->at(e2).size();
+        };
+        std::sort(active_nh_list.begin(), active_nh_list.end(), compare_active_buckets);
 
+        auto it = active_nh_list.begin();
         while(del_idx < bank_member_change.nhs_to_del.size())
         {
-            HashBuckets *hash_buckets = &(bank_fgnhg_map->at(bank_member_change.nhs_to_del[del_idx]));
-            for (uint32_t i = 0; i < hash_buckets->size(); i++)
+            HashBuckets *hash_buckets = &(fgnhg_bucket_map->at(bank_member_change.nhs_to_del[del_idx]));
+            for (uint32_t bkt_idx = 0; bkt_idx < hash_buckets->size(); bkt_idx++)
             {
-                NextHopKey round_robin_nh = bank_member_change.active_nhs[i %
-                    bank_member_change.active_nhs.size()];
+                if (it == active_nh_list.end())
+                {
+                    it = active_nh_list.begin();
+                    if (it == active_nh_list.end())
+                    {
+                        // this can neven happen
+                        SWSS_LOG_ERROR("%s Unexpected no more active NHs before adding the %zu buckets, del(%d), exp(%zu)",
+                                ipPrefix.to_string().c_str(),
+                                hash_buckets->size(), del_idx, act_nh_exp_size);
+                        return false;
+                    }
+                }
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: Would you be able to share an example scenario from either ut or hash buckets where the current algorithm breaks? It will be easier to evaluate the change in conjunction with that. 

If we found that the unequal hash buckets happen because some nh is already at size when we are at the beginning of the for loop did you try to move the active nh removal logic to the beginning of the loop as a solution? 
created_at: 2025-02-16T17:33:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -610,22 +651,72 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        bool ends_with_balance = num_nhs_with_one_more ? false : true;
+        auto active_nh_list = bank_member_change.active_nhs;
+        size_t act_nh_exp_size = exp_bucket_size;
+
+        if (!ends_with_balance)
+        {
+            act_nh_exp_size++;
+        }
 
+        // arrange the active nhs in increasing order of size of their hash bucket map
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: in theory we don't really need to sort the active nh in increasing order, because they just need to reach the expected size. What is the case where we ran into issues with the current algorithm 
created_at: 2025-02-16T17:34:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -610,22 +651,72 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        bool ends_with_balance = num_nhs_with_one_more ? false : true;
+        auto active_nh_list = bank_member_change.active_nhs;
+        size_t act_nh_exp_size = exp_bucket_size;
+
+        if (!ends_with_balance)
+        {
+            act_nh_exp_size++;
+        }
 
+        // arrange the active nhs in increasing order of size of their hash bucket map
+        // so that we add buckets to the nhs which have lower bucket size first
+        auto compare_active_buckets = [&](NextHopKey e1, NextHopKey e2) {
+            return fgnhg_bucket_map->at(e1).size() < fgnhg_bucket_map->at(e2).size();
+        };
+        std::sort(active_nh_list.begin(), active_nh_list.end(), compare_active_buckets);
 
+        auto it = active_nh_list.begin();
         while(del_idx < bank_member_change.nhs_to_del.size())
         {
-            HashBuckets *hash_buckets = &(bank_fgnhg_map->at(bank_member_change.nhs_to_del[del_idx]));
-            for (uint32_t i = 0; i < hash_buckets->size(); i++)
+            HashBuckets *hash_buckets = &(fgnhg_bucket_map->at(bank_member_change.nhs_to_del[del_idx]));
+            for (uint32_t bkt_idx = 0; bkt_idx < hash_buckets->size(); bkt_idx++)
             {
-                NextHopKey round_robin_nh = bank_member_change.active_nhs[i %
-                    bank_member_change.active_nhs.size()];
+                if (it == active_nh_list.end())
+                {
+                    it = active_nh_list.begin();
+                    if (it == active_nh_list.end())
+                    {
+                        // this can neven happen
+                        SWSS_LOG_ERROR("%s Unexpected no more active NHs before adding the %zu buckets, del(%d), exp(%zu)",
+                                ipPrefix.to_string().c_str(),
+                                hash_buckets->size(), del_idx, act_nh_exp_size);
+                        return false;
+                    }
+                }
+
+                // remove the active nhs that have reached their expected bucket size
+                while (fgnhg_bucket_map->at(*it).size() == act_nh_exp_size)
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: Why do we need this additional while loop logic to remove active_nhs? I believe the logic at the end of the loop already handles this.
created_at: 2025-02-16T17:43:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -999,120 +1119,160 @@ bool FgNhgOrch::setNewNhgMembers(FGNextHopGroupEntry &syncd_fg_route_entry, FgNh
 {
     SWSS_LOG_ENTER();
 
-    sai_status_t status;
-    bool isWarmReboot = false;
-    auto nexthopsMap = m_recoveryMap.find(ipPrefix.to_string());
-    for (uint32_t i = 0; i < fgNhgEntry->hash_bucket_indices.size(); i++) 
+    auto num_banks = fgNhgEntry->hash_bucket_indices.size();
+    std::vector<uint32_t> active_banks;
+    std::vector<uint32_t> inactive_banks;
+
+    for (uint32_t i = 0; i < num_banks; i++)
     {
-        uint32_t bank = i;
-        syncd_fg_route_entry.inactive_to_active_map[bank] = bank;
+        if (bank_member_changes[i].nhs_to_add.size() != 0)
+        {
+            active_banks.push_back(i);
+            SWSS_LOG_INFO("active bank %d num nhs %zu", i, bank_member_changes[i].nhs_to_add.size());
+        }
+        else
+        {
+            inactive_banks.push_back(i);
+            SWSS_LOG_INFO("inactive bank %d", i);
+        }
+
+        // add a new fgnh group map if this is a new bank
         if (i + 1 > syncd_fg_route_entry.syncd_fgnhg_map.size())
         {
             syncd_fg_route_entry.syncd_fgnhg_map.push_back(FGNextHopGroupMap());
         }
+    }
 
-        if (bank_member_changes[i].nhs_to_add.size() == 0)
-        {
-            /* Case where bank is empty */
-            for (uint32_t active_bank = 0; active_bank < bank_member_changes.size(); active_bank++)
-            {
-                if (bank_member_changes[active_bank].nhs_to_add.size() != 0)
-                {
-                    bank = active_bank;
-                    syncd_fg_route_entry.inactive_to_active_map[i] = active_bank;
-                    break;
-                }
-            }
-            SWSS_LOG_NOTICE("Bank# %d of FG next-hops is down for prefix %s", 
-                    i, ipPrefix.to_string().c_str());
-        } 
 
-        if (bank_member_changes[bank].nhs_to_add.size() == 0)
+    /* Case where all banks are empty, we let retry logic(upon rv false) take care of this scenario */
+    if (active_banks.size() == 0)
+    {
+        SWSS_LOG_INFO("Found no next-hops to add, skipping");
+        return false;
+    }
+    // spray the nhs of active banks into active bank's buckets
+    for (auto active_bank : active_banks)
+    {
+        syncd_fg_route_entry.inactive_to_active_map[active_bank] = active_bank;
+        if (!sprayBankNhgMembers(syncd_fg_route_entry, ipPrefix,
+                fgNhgEntry->hash_bucket_indices[active_bank], fgNhgEntry,
+                active_bank, bank_member_changes[active_bank],
+                nhopgroup_members_set))
         {
-            /* Case where all banks are empty, we let retry logic(upon rv false) take care of this scenario */
-            SWSS_LOG_INFO("Found no next-hops to add, skipping");
             return false;
         }
+    }
 
-        // recover state before warm reboot
-        if (nexthopsMap != m_recoveryMap.end())
+    // spray the nhs of active banks into inactive bank's buckets
+    for (uint32_t i = 0; i < inactive_banks.size(); i++)
+    {
+        auto active_bank = active_banks[i % active_banks.size()];
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: So if I understand correctly, the intent of the changes in the function setNewNhgMembers is to be able to loadbalance inactive_to_active_map to multiple active banks instead of just setting it to the 1st bank as it is done today, and the rest of the changes are code readability/variable name improvements. 
created_at: 2025-02-16T20:42:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1162,7 +1326,21 @@ bool FgNhgOrch::isRouteFineGrained(sai_object_id_t vrf_id, const IpPrefix &ipPre
                 }
             }
         }
+        if (static_fgnhg)
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: do we need this logic? I think the logic above covers all the cases? 
created_at: 2025-02-16T21:00:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1227,6 +1405,15 @@ bool FgNhgOrch::setFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const
         return true;
     }
 
+    if (!fgNhgEntry)
+    {
+        SWSS_LOG_INFO("fgNhgOrch got a route addition %s:%s with %s",
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: This should never happen because routeorch checks for ```if (isRouteFineGrained)``` first, and I am not aware of any case where this has happened before. Makes sense to have this check for defensive programming and if this ever happens it is an error condition, can you change this log to error? 
created_at: 2025-02-16T21:16:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1244,11 +1431,73 @@ bool FgNhgOrch::setFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const
     {
         /* Only happens the 1st time when hash_bucket_indices are not initialized
          */
-        for (auto it : fgNhgEntry->next_hops)
+        if (fgNhgEntry->nhg_mode == DYNAMIC_FGNHG) //NEW
+        {
+            //bank_member_changes.resize(fgNhgEntry->max_next_hops, BankMemberChanges());
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: remove comment?
created_at: 2025-02-16T21:18:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1546,9 +1803,37 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t)
                     SWSS_LOG_WARN("Received unsupported match_mode %s, defaulted to route-based",
                                     fvValue(i).c_str());
                 }
+            } 
+            else if (fvField(i) == "nhg_mode")
+            {
+                if (fvValue(i) == "dynamic-nhg")
+                {
+                    nhg_mode = DYNAMIC_FGNHG;
+                }
+                else if (fvValue(i) != "static-nhg")
+                {
+                    SWSS_LOG_WARN("Received unsupported match_mode %s, defaulted to static-nhg",
+                                    fvValue(i).c_str());
+                }
+            } 
+            else if (fvField(i) == "max_next_hops")
+            {
+                max_next_hops = stoi(fvValue(i));
             }
         }
 
+        // Nexthop-based match_mode is not supported for dynamic_fgnhg
+        if (nhg_mode == DYNAMIC_FGNHG && match_mode == NEXTHOP_BASED) 
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: Looks like we only log an error in this case, should we default/override to static FGNHG if this happens? 
created_at: 2025-02-16T21:28:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1640,6 +1940,19 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t)
             return false;
         }
 
+        // In the dynamic_nhg mode, only 1 FG_NHG_PREFIX is allowed for each FG_NHG. If ip_prefix 
+        // doesn't match with the existing prefixes in the fgNhgEntry, then return false.
+        if (fgNhg_entry->second.nhg_mode == DYNAMIC_FGNHG)
+        {
+            if (fgNhg_entry->second.prefixes.size() != 0)
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: why is it the case that there can only be 1 prefix per FG_NHG? I think allowing multiple prefixes is possible associated with the same set of FG_NHG config. FG_NHG is intended to be shared config across a bunch of prefixes/next-hop group definitions.
created_at: 2025-02-16T21:34:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -56,25 +57,34 @@ typedef std::unordered_map<string, std::vector<IpAddress>> Links;
 enum FGMatchMode
 {
     ROUTE_BASED,
-    NEXTHOP_BASED
+    NEXTHOP_BASED,
+    DYNAMIC_ROUTE_BASED
 };
+enum FGNexthopMode
+{
+    STATIC_FGNHG,
+    DYNAMIC_FGNHG
+};
+
 /* Store the indices occupied by a bank */
 typedef struct
 {
     uint32_t start_index;
     uint32_t end_index;
-} BankIndexRange;
+} HashIndexRange;
path: orchagent/fgnhgorch.h
user: login: manamand2020
body: To be clear I will rename this to BankHashIndexRange, hope that is more intuitive.
created_at: 2025-02-17T01:23:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -15,9 +15,10 @@
 #include <map>
 
 typedef uint32_t Bank;
+typedef uint32_t HashBucketIdx;
 typedef std::set<NextHopKey> ActiveNextHops;
-typedef std::vector<sai_object_id_t> FGNextHopGroupMembers;
-typedef std::vector<uint32_t> HashBuckets;
+typedef std::map<HashBucketIdx, sai_object_id_t> FGNextHopGroupMembers;
path: orchagent/fgnhgorch.h
user: login: manamand2020
body: After we separated the logic to handle per bank with sprayBankNhgMembers the BucketHashIndex start may not necessarily start from 0 and it depends on order.
created_at: 2025-02-17T01:44:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -999,120 +1119,160 @@ bool FgNhgOrch::setNewNhgMembers(FGNextHopGroupEntry &syncd_fg_route_entry, FgNh
 {
     SWSS_LOG_ENTER();
 
-    sai_status_t status;
-    bool isWarmReboot = false;
-    auto nexthopsMap = m_recoveryMap.find(ipPrefix.to_string());
-    for (uint32_t i = 0; i < fgNhgEntry->hash_bucket_indices.size(); i++) 
+    auto num_banks = fgNhgEntry->hash_bucket_indices.size();
+    std::vector<uint32_t> active_banks;
+    std::vector<uint32_t> inactive_banks;
+
+    for (uint32_t i = 0; i < num_banks; i++)
     {
-        uint32_t bank = i;
-        syncd_fg_route_entry.inactive_to_active_map[bank] = bank;
+        if (bank_member_changes[i].nhs_to_add.size() != 0)
+        {
+            active_banks.push_back(i);
+            SWSS_LOG_INFO("active bank %d num nhs %zu", i, bank_member_changes[i].nhs_to_add.size());
+        }
+        else
+        {
+            inactive_banks.push_back(i);
+            SWSS_LOG_INFO("inactive bank %d", i);
+        }
+
+        // add a new fgnh group map if this is a new bank
         if (i + 1 > syncd_fg_route_entry.syncd_fgnhg_map.size())
         {
             syncd_fg_route_entry.syncd_fgnhg_map.push_back(FGNextHopGroupMap());
         }
+    }
 
-        if (bank_member_changes[i].nhs_to_add.size() == 0)
-        {
-            /* Case where bank is empty */
-            for (uint32_t active_bank = 0; active_bank < bank_member_changes.size(); active_bank++)
-            {
-                if (bank_member_changes[active_bank].nhs_to_add.size() != 0)
-                {
-                    bank = active_bank;
-                    syncd_fg_route_entry.inactive_to_active_map[i] = active_bank;
-                    break;
-                }
-            }
-            SWSS_LOG_NOTICE("Bank# %d of FG next-hops is down for prefix %s", 
-                    i, ipPrefix.to_string().c_str());
-        } 
 
-        if (bank_member_changes[bank].nhs_to_add.size() == 0)
+    /* Case where all banks are empty, we let retry logic(upon rv false) take care of this scenario */
+    if (active_banks.size() == 0)
+    {
+        SWSS_LOG_INFO("Found no next-hops to add, skipping");
+        return false;
+    }
+    // spray the nhs of active banks into active bank's buckets
+    for (auto active_bank : active_banks)
+    {
+        syncd_fg_route_entry.inactive_to_active_map[active_bank] = active_bank;
+        if (!sprayBankNhgMembers(syncd_fg_route_entry, ipPrefix,
+                fgNhgEntry->hash_bucket_indices[active_bank], fgNhgEntry,
+                active_bank, bank_member_changes[active_bank],
+                nhopgroup_members_set))
         {
-            /* Case where all banks are empty, we let retry logic(upon rv false) take care of this scenario */
-            SWSS_LOG_INFO("Found no next-hops to add, skipping");
             return false;
         }
+    }
 
-        // recover state before warm reboot
-        if (nexthopsMap != m_recoveryMap.end())
+    // spray the nhs of active banks into inactive bank's buckets
+    for (uint32_t i = 0; i < inactive_banks.size(); i++)
+    {
+        auto active_bank = active_banks[i % active_banks.size()];
path: orchagent/fgnhgorch.cpp
user: login: manamand2020
body: Yes
created_at: 2025-02-17T03:41:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1244,11 +1431,73 @@ bool FgNhgOrch::setFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const
     {
         /* Only happens the 1st time when hash_bucket_indices are not initialized
          */
-        for (auto it : fgNhgEntry->next_hops)
+        if (fgNhgEntry->nhg_mode == DYNAMIC_FGNHG) //NEW
+        {
+            //bank_member_changes.resize(fgNhgEntry->max_next_hops, BankMemberChanges());
path: orchagent/fgnhgorch.cpp
user: login: manamand2020
body: Yes
created_at: 2025-02-18T06:16:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1640,6 +1940,19 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t)
             return false;
         }
 
+        // In the dynamic_nhg mode, only 1 FG_NHG_PREFIX is allowed for each FG_NHG. If ip_prefix 
+        // doesn't match with the existing prefixes in the fgNhgEntry, then return false.
+        if (fgNhg_entry->second.nhg_mode == DYNAMIC_FGNHG)
+        {
+            if (fgNhg_entry->second.prefixes.size() != 0)
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: In this case, list of nexthop IPs is not statically configured, and it's quite possible that routing protocol will push a different set of nexthop IPs for each of the prefixes and we don't have a mechanism to create a common group out of them. So, we have an explicit check to avoid that configuration.
created_at: 2025-02-19T06:14:37Z
author_association: MEMBER
diff_hunk: @@ -1546,9 +1803,37 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t)
                     SWSS_LOG_WARN("Received unsupported match_mode %s, defaulted to route-based",
                                     fvValue(i).c_str());
                 }
+            } 
+            else if (fvField(i) == "nhg_mode")
+            {
+                if (fvValue(i) == "dynamic-nhg")
+                {
+                    nhg_mode = DYNAMIC_FGNHG;
+                }
+                else if (fvValue(i) != "static-nhg")
+                {
+                    SWSS_LOG_WARN("Received unsupported match_mode %s, defaulted to static-nhg",
+                                    fvValue(i).c_str());
+                }
+            } 
+            else if (fvField(i) == "max_next_hops")
+            {
+                max_next_hops = stoi(fvValue(i));
             }
         }
 
+        // Nexthop-based match_mode is not supported for dynamic_fgnhg
+        if (nhg_mode == DYNAMIC_FGNHG && match_mode == NEXTHOP_BASED) 
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: Replaced this combination of nhg_mode and match_mode with match_mode==prefix_based; so this check has been removed now.
created_at: 2025-02-19T06:16:13Z
author_association: MEMBER
diff_hunk: @@ -610,22 +651,72 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        bool ends_with_balance = num_nhs_with_one_more ? false : true;
+        auto active_nh_list = bank_member_change.active_nhs;
+        size_t act_nh_exp_size = exp_bucket_size;
+
+        if (!ends_with_balance)
+        {
+            act_nh_exp_size++;
+        }
 
+        // arrange the active nhs in increasing order of size of their hash bucket map
+        // so that we add buckets to the nhs which have lower bucket size first
+        auto compare_active_buckets = [&](NextHopKey e1, NextHopKey e2) {
+            return fgnhg_bucket_map->at(e1).size() < fgnhg_bucket_map->at(e2).size();
+        };
+        std::sort(active_nh_list.begin(), active_nh_list.end(), compare_active_buckets);
 
+        auto it = active_nh_list.begin();
         while(del_idx < bank_member_change.nhs_to_del.size())
         {
-            HashBuckets *hash_buckets = &(bank_fgnhg_map->at(bank_member_change.nhs_to_del[del_idx]));
-            for (uint32_t i = 0; i < hash_buckets->size(); i++)
+            HashBuckets *hash_buckets = &(fgnhg_bucket_map->at(bank_member_change.nhs_to_del[del_idx]));
+            for (uint32_t bkt_idx = 0; bkt_idx < hash_buckets->size(); bkt_idx++)
             {
-                NextHopKey round_robin_nh = bank_member_change.active_nhs[i %
-                    bank_member_change.active_nhs.size()];
+                if (it == active_nh_list.end())
+                {
+                    it = active_nh_list.begin();
+                    if (it == active_nh_list.end())
+                    {
+                        // this can neven happen
+                        SWSS_LOG_ERROR("%s Unexpected no more active NHs before adding the %zu buckets, del(%d), exp(%zu)",
+                                ipPrefix.to_string().c_str(),
+                                hash_buckets->size(), del_idx, act_nh_exp_size);
+                        return false;
+                    }
+                }
path: orchagent/fgnhgorch.cpp
user: login: manamand2020
body: This is an example where we started with 16 NHs and 128 buckets. 2 NHs are withdrawn here, so now each NH should get either 9 or 10 hash buckets. But look at the NH 2001::20:1:0:3 and 2001::20:1:0:5 in bold which got only 8 buckets.

6100:1::1/128    2001::20:1:0:10  7   8   9   10  11  12  13  14
                                  15
6100:1::1/128    2001::20:1:0:2   0   71  120 121 122 123 124 125
                                  126 127
**6100:1::1/128    2001::20:1:0:3   112 113 114 115 116 117 118 119**
6100:1::1/128    2001::20:1:0:4   1   70  104 105 106 107 108 109
                                  110 111
**6100:1::1/128    2001::20:1:0:5   96  97  98  99  100 101 102 103**
6100:1::1/128    2001::20:1:0:6   2   69  88  89  90  91  92  93
                                  94  95
6100:1::1/128    2001::20:1:0:7   68  80  81  82  83  84  85  86
                                  87
6100:1::1/128    2001::20:1:0:8   3   72  73  74  75  76  77  78
                                  79
6100:1::1/128    2001::20:1:0:a   4   56  57  58  59  60  61  62
                                  63  67
6100:1::1/128    2001::20:1:0:b   48  49  50  51  52  53  54  55
                                  66
6100:1::1/128    2001::20:1:0:c   5   40  41  42  43  44  45  46
                                  47
6100:1::1/128    2001::20:1:0:d   32  33  34  35  36  37  38  39
                                  65
6100:1::1/128    2001::20:1:0:e   6   24  25  26  27  28  29  30
                                  31
6100:1::1/128    2001::20:1:0:f   16  17  18  19  20  21  22  23
                                  64

created_at: 2025-03-15T00:56:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -152,81 +152,112 @@ bool FgNhgOrch::bake()
 void FgNhgOrch::calculateBankHashBucketStartIndices(FgNhgEntry *fgNhgEntry)
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: I've reverted variable name changes.
created_at: 2025-03-26T22:48:47Z
author_association: MEMBER
diff_hunk: @@ -610,22 +651,72 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        bool ends_with_balance = num_nhs_with_one_more ? false : true;
+        auto active_nh_list = bank_member_change.active_nhs;
+        size_t act_nh_exp_size = exp_bucket_size;
+
+        if (!ends_with_balance)
+        {
+            act_nh_exp_size++;
+        }
 
+        // arrange the active nhs in increasing order of size of their hash bucket map
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: Here is an example for a case where I took this PR and simply commented out sorting for both add and del cases. I started with 16 nexthops and 256 buckets and then started shutting down one interface at a time. Transitions 16->15->14->13 went fine and buckets were evenly balanced with nexthops with a max difference of 1. On the 13->12 transition, when nexthop 172.17.3.2 went down, it's 20 buckets were redistributed to remaining nexthops without considering which nexthops already had more buckets than others. So, Some nexthops like 172.17.11.2 ended up with 2 more buckets than others like 172.17.12.2. 

```
**13 active nexthops**
FG NHG Prefix    Next Hop     Hash buckets
---------------  -----------  -------------------------------
10.10.10.0/24    172.17.10.2  1   10  23  36  49  62  75  88
                              93  101 105 114 117 127 140 153
                              166 179 192 205
10.10.10.0/24    172.17.11.2  11  24  37  50  63  76  89  102
                              106 115 118 128 130 141 154 167
                              180 183 193 206
10.10.10.0/24    172.17.12.2  12  25  38  51  64  77  90  103
                              116 119 129 131 142 143 155 168
                              181 194 207
10.10.10.0/24    172.17.13.2  132 144 156 238 239 240 243 244
                              245 246 247 248 249 250 251 252
                              253 254 255
10.10.10.0/24    172.17.14.2  145 157 169 222 223 225 226 227
                              228 229 230 231 232 233 234 235
                              236 237 242
10.10.10.0/24    172.17.15.2  158 170 182 208 209 210 211 212
                              213 214 215 216 217 218 219 220
                              221 224 241
10.10.10.0/24    172.17.3.2   2   3   14  16  26  29  42  55
                              68  81  94  107 120 133 146 159
                              172 185 196 198
10.10.10.0/24    172.17.4.2   0   4   15  17  27  30  39  43
                              56  69  82  95  108 121 134 147
                              160 173 186 199
10.10.10.0/24    172.17.5.2   5   18  28  31  40  44  52  57
                              70  83  96  109 122 135 148 161
                              174 187 195 200
10.10.10.0/24    172.17.6.2   6   19  32  41  45  53  58  65
                              71  84  97  110 123 136 149 162
                              171 175 188 201
10.10.10.0/24    172.17.7.2   7   20  33  46  54  59  66  72
                              78  85  98  111 124 137 150 163
                              176 184 189 202
10.10.10.0/24    172.17.8.2   8   21  34  47  60  67  73  79
                              86  91  99  112 125 138 151 164
                              177 190 197 203
10.10.10.0/24    172.17.9.2   9   13  22  35  48  61  74  80
                              87  92  100 104 113 126 139 152
                              165 178 191 204
                              
**After 172.17.3.2 went down**
FG NHG Prefix    Next Hop     Hash buckets
---------------  -----------  -------------------------------
10.10.10.0/24    172.17.10.2  1   10  23  36  49  62  75  81
                              88  93  101 105 114 117 127 140
                              153 166 179 192 196 205
10.10.10.0/24    172.17.11.2  2   11  24  37  50  63  76  89
                              94  102 106 115 118 128 130 141
                              154 167 180 183 193 206
10.10.10.0/24    172.17.12.2  12  25  38  51  64  77  90  103
                              107 116 119 129 131 142 143 155
                              168 181 194 207
10.10.10.0/24    172.17.13.2  120 132 144 156 238 239 240 243
                              244 245 246 247 248 249 250 251
                              252 253 254 255
10.10.10.0/24    172.17.14.2  133 145 157 169 222 223 225 226
                              227 228 229 230 231 232 233 234
                              235 236 237 242
10.10.10.0/24    172.17.15.2  146 158 170 182 208 209 210 211
                              212 213 214 215 216 217 218 219
                              220 221 224 241
10.10.10.0/24    172.17.4.2   0   3   4   15  17  27  30  39
                              43  56  69  82  95  108 121 134
                              147 159 160 173 186 199
10.10.10.0/24    172.17.5.2   5   16  18  28  31  40  44  52
                              57  70  83  96  109 122 135 148
                              161 172 174 187 195 200
10.10.10.0/24    172.17.6.2   6   19  29  32  41  45  53  58
                              65  71  84  97  110 123 136 149
                              162 171 175 185 188 201
10.10.10.0/24    172.17.7.2   7   20  33  42  46  54  59  66
                              72  78  85  98  111 124 137 150
                              163 176 184 189 198 202
10.10.10.0/24    172.17.8.2   8   21  26  34  47  55  60  67
                              73  79  86  91  99  112 125 138
                              151 164 177 190 197 203
10.10.10.0/24    172.17.9.2   9   13  14  22  35  48  61  68
                              74  80  87  92  100 104 113 126
                              139 152 165 178 191 204
```

I noticed similar imbalance on transitions 10>9 and 6->5. We don't see the same problem after nexthop list is sorted.
created_at: 2025-03-27T00:24:13Z
author_association: MEMBER
diff_hunk: @@ -566,19 +605,21 @@ bool FgNhgOrch::invalidNextHopInNextHopGroup(const NextHopKey& nexthop)
  * Thereby achieving consistent and layered hashing.
  */
 bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_route_entry, FgNhgEntry *fgNhgEntry,
-        uint32_t bank, uint32_t syncd_bank, std::vector<BankMemberChanges> bank_member_changes, 
+        uint32_t syncd_bank, BankMemberChanges bank_member_change,
         std::map<NextHopKey,sai_object_id_t> &nhopgroup_members_set, const IpPrefix &ipPrefix)
 {
     SWSS_LOG_ENTER();
 
-    BankMemberChanges bank_member_change = bank_member_changes[bank];
     uint32_t add_idx = 0, del_idx = 0;
-    FGNextHopGroupMap *bank_fgnhg_map = &(syncd_fg_route_entry->syncd_fgnhg_map[syncd_bank]);
+    FGNextHopGroupMap *fgnhg_bucket_map = &(syncd_fg_route_entry->syncd_fgnhg_map[syncd_bank]);
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: Done.
created_at: 2025-03-27T01:17:51Z
author_association: MEMBER
diff_hunk: @@ -1227,6 +1405,15 @@ bool FgNhgOrch::setFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const
         return true;
     }
 
+    if (!fgNhgEntry)
+    {
+        SWSS_LOG_INFO("fgNhgOrch got a route addition %s:%s with %s",
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: Changed to LOG_ERROR as requested.
created_at: 2025-03-27T01:19:04Z
author_association: MEMBER
diff_hunk: @@ -610,22 +651,72 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        bool ends_with_balance = num_nhs_with_one_more ? false : true;
+        auto active_nh_list = bank_member_change.active_nhs;
+        size_t act_nh_exp_size = exp_bucket_size;
+
+        if (!ends_with_balance)
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: Done.
created_at: 2025-03-27T01:21:13Z
author_association: MEMBER
diff_hunk: @@ -56,25 +57,34 @@ typedef std::unordered_map<string, std::vector<IpAddress>> Links;
 enum FGMatchMode
 {
     ROUTE_BASED,
-    NEXTHOP_BASED
+    NEXTHOP_BASED,
+    DYNAMIC_ROUTE_BASED
 };
+enum FGNexthopMode
+{
+    STATIC_FGNHG,
+    DYNAMIC_FGNHG
+};
+
 /* Store the indices occupied by a bank */
 typedef struct
 {
     uint32_t start_index;
     uint32_t end_index;
-} BankIndexRange;
+} HashIndexRange;
path: orchagent/fgnhgorch.h
user: login: anish-n
body: Sounds good!
created_at: 2025-04-05T20:27:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -15,9 +15,10 @@
 #include <map>
 
 typedef uint32_t Bank;
+typedef uint32_t HashBucketIdx;
 typedef std::set<NextHopKey> ActiveNextHops;
-typedef std::vector<sai_object_id_t> FGNextHopGroupMembers;
-typedef std::vector<uint32_t> HashBuckets;
+typedef std::map<HashBucketIdx, sai_object_id_t> FGNextHopGroupMembers;
path: orchagent/fgnhgorch.h
user: login: anish-n
body: It does not have to start at 0 for us to use the vector based implementation, we can always default init the vector with size = num of buckets, and NULL elems and then you can put the members as usual. I think the only problem you are hitting is that you can't do push_back anymore because you are writing to member n instead of starting from 0 like in the original implementation. Let me know how that sounds. 
created_at: 2025-04-05T20:30:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -267,8 +274,10 @@ def program_route_and_validate_fine_grained_ecmp(app_db, asic_db, state_db, ip_t
     ps = swsscommon.ProducerStateTable(app_db, ROUTE_TB)
     fvs = swsscommon.FieldValuePairs([("nexthop", ips), ("ifname", ifs)])
     ps.set(fg_nhg_prefix, fvs)
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                          fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    time.sleep(1)
path: tests/test_fgnhg.py
user: login: anish-n
body: why are we adding a sleep here, validate_fine_grained_asic_n_state_db_entries already has polling and sleep.
created_at: 2025-04-06T18:19:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -376,134 +391,158 @@ def fine_grained_ecmp_base_test(dvs, match_mode):
     dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03")
     dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
     dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+    time.sleep(1)
path: tests/test_fgnhg.py
user: login: anish-n
body: remove sleep
created_at: 2025-04-06T18:21:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -376,134 +391,158 @@ def fine_grained_ecmp_base_test(dvs, match_mode):
     dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03")
     dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
     dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+    time.sleep(1)
     asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 5)
 
     asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
     nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
 
     nh_oid_map = get_nh_oid_map(asic_db)
+    memb_dict = {}
 
     ### Test scenarios with bank 0 having 0 members up and only bank 1 having members
     # ARP is not resolved for 10.0.0.7, so fg nhg should be created without 10.0.0.7
     nh_memb_exp_count = {"10.0.0.9":30,"10.0.0.11":30}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Resolve ARP for 10.0.0.7
     asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
     dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    time.sleep(1)
path: tests/test_fgnhg.py
user: login: anish-n
body: remove
created_at: 2025-04-06T18:22:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -610,22 +651,72 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        bool ends_with_balance = num_nhs_with_one_more ? false : true;
+        auto active_nh_list = bank_member_change.active_nhs;
+        size_t act_nh_exp_size = exp_bucket_size;
+
+        if (!ends_with_balance)
+        {
+            act_nh_exp_size++;
+        }
 
+        // arrange the active nhs in increasing order of size of their hash bucket map
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: I have 3 observations related to the behavior you are seeing:
1. 172.17.11.2 contains 22 buckets, spread over 12 nhs, this is the right number of buckets for 4 nhs to have.
2. There should be 4 nhs with 22, and 8 nhs with 21, but for your case what I am seeing is that there are 8 with 22, and 4 with 20.
3. The sorting is a bandaid, the real root cause lies in the newly added code, wherein: You are not decrementing num_nhs_with_one_more in the while loop where you check for those which have reached act_nh_exp_size, which is why this can occur. Lets remove the sorting and fix the code bug.
created_at: 2025-04-06T19:15:27Z
author_association: CONTRIBUTOR
diff_hunk: @@ -610,22 +651,72 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        bool ends_with_balance = num_nhs_with_one_more ? false : true;
+        auto active_nh_list = bank_member_change.active_nhs;
+        size_t act_nh_exp_size = exp_bucket_size;
+
+        if (!ends_with_balance)
+        {
+            act_nh_exp_size++;
+        }
 
+        // arrange the active nhs in increasing order of size of their hash bucket map
+        // so that we add buckets to the nhs which have lower bucket size first
+        auto compare_active_buckets = [&](NextHopKey e1, NextHopKey e2) {
+            return fgnhg_bucket_map->at(e1).size() < fgnhg_bucket_map->at(e2).size();
+        };
+        std::sort(active_nh_list.begin(), active_nh_list.end(), compare_active_buckets);
 
+        auto it = active_nh_list.begin();
         while(del_idx < bank_member_change.nhs_to_del.size())
         {
-            HashBuckets *hash_buckets = &(bank_fgnhg_map->at(bank_member_change.nhs_to_del[del_idx]));
-            for (uint32_t i = 0; i < hash_buckets->size(); i++)
+            HashBuckets *hash_buckets = &(fgnhg_bucket_map->at(bank_member_change.nhs_to_del[del_idx]));
+            for (uint32_t bkt_idx = 0; bkt_idx < hash_buckets->size(); bkt_idx++)
             {
-                NextHopKey round_robin_nh = bank_member_change.active_nhs[i %
-                    bank_member_change.active_nhs.size()];
+                if (it == active_nh_list.end())
+                {
+                    it = active_nh_list.begin();
+                    if (it == active_nh_list.end())
+                    {
+                        // this can neven happen
+                        SWSS_LOG_ERROR("%s Unexpected no more active NHs before adding the %zu buckets, del(%d), exp(%zu)",
+                                ipPrefix.to_string().c_str(),
+                                hash_buckets->size(), del_idx, act_nh_exp_size);
+                        return false;
+                    }
+                }
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: Ok I see, understood, so my initial comment as follows is the reasoning for the change: "If we found that the unequal hash buckets happen because some nh is already at size when we are at the beginning of the for loop did you try to move the active nh removal logic to the beginning of the loop as a solution?"

I think the newly introduced logic to make fixes is quite complicated because we are doing all of the following as per the new logic:
1. Iterator check and move
2. While loop + check and delete elements which have reached act_nh_exp_size(note there is a bug in the added code)
3. Write sai change and store new elem in bucket
4. Once again perform an if checking and delete elements
5. Multiple nested loops

Here is what I would suggest, 1, 2 and 4 should be combined and happen right at the beginning in an inner while loop. With so many things going on here, there is a likelihood of more bugs showing up(like the one discovered and called out above).
created_at: 2025-04-06T19:55:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -610,64 +651,112 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        auto active_nh_list = bank_member_change.active_nhs;
+        size_t act_nh_exp_size = exp_bucket_size;
 
+        if (num_nhs_with_one_more > 0)
+        {
+            act_nh_exp_size++;
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: Can we instead just use exp_bucket_size + 1 where needed, its used in other places in the code as well. We should unify to either using "exp_bucket_size +1" or use a var to represent that value everywhere, right now I see that we are using both conventions.
created_at: 2025-04-06T19:59:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -148,7 +162,7 @@ class FgNhgOrch : public Orch, public Observer
                     uint32_t bank, std::vector<BankMemberChanges> bank_member_changes,
                     std::map<NextHopKey,sai_object_id_t> &nhopgroup_members_set, const IpPrefix&);
     void calculateBankHashBucketStartIndices(FgNhgEntry *fgNhgEntry);
-    void setStateDbRouteEntry(const IpPrefix&, uint32_t index, NextHopKey nextHop);
+    void setStateDbRouteEntry(const IpPrefix&, uint32_t bucket_index, NextHopKey nextHop);
path: orchagent/fgnhgorch.h
user: login: anish-n
body: function definition is inconsistent between .h and .cpp file. Here the param name is set to bucket_index, whereas in the cpp it is named index. 
created_at: 2025-04-06T20:51:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -718,9 +818,16 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
                 }
                 else
                 {
-                    uint32_t last_elem = map_entry->at((*map_entry).size() - 1);
+                    if ((*map_entry).size() == exp_size_before_bkt_rem)
+                    {
+                        SWSS_LOG_INFO("%s Buckets size already at %d, don't remove more",
+                                it->to_string().c_str(), exp_size_before_bkt_rem);
+                        it = active_nh_list.erase(it);
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: There are 2 bugs introduced by this new change:
1. It will result in double erasal from active_nh_list(here as well as deletion per the original logic below), thereby causing you to delete 2 entries in 1 loop, and removing an entry you should not be deleting from the active_nh_list.
2. You are still allocating a bucket from this entry despite the delete performed here, so don't think this logic is doing what it is intended to do. 
created_at: 2025-04-06T21:21:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -686,12 +775,21 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % total_nhs);
         uint32_t num_nhs_with_eq_to_exp = total_nhs - num_nhs_with_one_more;
         uint32_t add_nh_exp_bucket_size = exp_bucket_size;
+        uint32_t exp_size_before_bkt_rem = exp_bucket_size;
+
+        auto active_nh_list = bank_member_change.active_nhs;
+        auto compare_active_buckets = [&](NextHopKey e1, NextHopKey e2) {
+            return bank_fgnhg_map->at(e1).size() > bank_fgnhg_map->at(e2).size();
+        };
+        // arrange the active nhs in decreasing order of size of their hash bucket map
+        // so that we replace buckets of the active nhs which have higher bucket size first
+        std::sort(active_nh_list.begin(), active_nh_list.end(), compare_active_buckets);
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: remove sorting
created_at: 2025-04-06T21:23:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -686,12 +775,21 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % total_nhs);
         uint32_t num_nhs_with_eq_to_exp = total_nhs - num_nhs_with_one_more;
         uint32_t add_nh_exp_bucket_size = exp_bucket_size;
+        uint32_t exp_size_before_bkt_rem = exp_bucket_size;
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: remove variable and use exp_bucket_size itself
created_at: 2025-04-06T21:28:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -968,20 +1085,22 @@ bool FgNhgOrch::computeAndSetHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
                 (bank_member_changes[bank_idx].nhs_to_add.size() != 0 &&
                  bank_member_changes[bank_idx].nhs_to_del.size() != 0))
         {
+            SWSS_LOG_INFO("active nhs in bank %u", bank_idx);
             /* Active bank is is determined by there being active nhs on the bank OR
              * an edge case where all active_nhs went down(nhs_to_del > 0) BUT
-             * simultaneously, nhs were added(nhs_to_add > 0). 
+             * simultaneously, nhs were added(nhs_to_add > 0).
              * Route this to fn which deals with active banks
              */
-            if (!setActiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, 
-                        bank_idx, bank_idx, bank_member_changes, nhopgroup_members_set, ipPrefix))
+            if (!setActiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry,
+                        bank_idx, bank_member_changes[bank_idx], nhopgroup_members_set, ipPrefix))
             {
                 return false;
             }
         }
         else
         {
-            if (!setInactiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, 
+            SWSS_LOG_INFO("No active nhs in bank %u", bank_idx);
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: Log exists in setInactiveBankHashBucketChanges, can we remove this? 
created_at: 2025-04-06T21:41:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -204,29 +219,45 @@ void FgNhgOrch::setStateDbRouteEntry(const IpPrefix &ipPrefix, uint32_t index, N
     // Write to StateDb
     std::vector<FieldValueTuple> fvs;
 
-    // check if profile already exists - if yes - skip creation
+    // get the existing buckets from warm restart state table
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: There is another PR which is attempting to solve the state db key sequencing problem. https://github.com/sonic-net/sonic-swss/pull/3566. 

I suggested a change there to modify the code to use hset instead of get + set as it is currently implemented. I think that hset is a simpler approach compared to creating a different data structure to track the index of the element inside redis(as is done here). Please try that and see if it works. In any case I have suggested the same change on the other PR linked above which is trying to solve this problem as well.
created_at: 2025-04-06T22:44:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1162,7 +1326,21 @@ bool FgNhgOrch::isRouteFineGrained(sai_object_id_t vrf_id, const IpPrefix &ipPre
                 }
             }
         }
+        if (static_fgnhg)
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: Lets try to limit changes and logging here as much as possible, because this fn gets called from routeorch and is present on the path of all route creations. I would suggest keeping this function as-is with no changes, because it seems like it would work as-is, do we see any case where this check is needed?
created_at: 2025-04-06T23:06:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -610,22 +651,72 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        bool ends_with_balance = num_nhs_with_one_more ? false : true;
+        auto active_nh_list = bank_member_change.active_nhs;
+        size_t act_nh_exp_size = exp_bucket_size;
+
+        if (!ends_with_balance)
+        {
+            act_nh_exp_size++;
+        }
 
+        // arrange the active nhs in increasing order of size of their hash bucket map
+        // so that we add buckets to the nhs which have lower bucket size first
+        auto compare_active_buckets = [&](NextHopKey e1, NextHopKey e2) {
+            return fgnhg_bucket_map->at(e1).size() < fgnhg_bucket_map->at(e2).size();
+        };
+        std::sort(active_nh_list.begin(), active_nh_list.end(), compare_active_buckets);
 
+        auto it = active_nh_list.begin();
         while(del_idx < bank_member_change.nhs_to_del.size())
         {
-            HashBuckets *hash_buckets = &(bank_fgnhg_map->at(bank_member_change.nhs_to_del[del_idx]));
-            for (uint32_t i = 0; i < hash_buckets->size(); i++)
+            HashBuckets *hash_buckets = &(fgnhg_bucket_map->at(bank_member_change.nhs_to_del[del_idx]));
+            for (uint32_t bkt_idx = 0; bkt_idx < hash_buckets->size(); bkt_idx++)
             {
-                NextHopKey round_robin_nh = bank_member_change.active_nhs[i %
-                    bank_member_change.active_nhs.size()];
+                if (it == active_nh_list.end())
+                {
+                    it = active_nh_list.begin();
+                    if (it == active_nh_list.end())
+                    {
+                        // this can neven happen
+                        SWSS_LOG_ERROR("%s Unexpected no more active NHs before adding the %zu buckets, del(%d), exp(%zu)",
+                                ipPrefix.to_string().c_str(),
+                                hash_buckets->size(), del_idx, act_nh_exp_size);
+                        return false;
+                    }
+                }
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: I've rewritten the logic so we first check for all the conditions before making any changes to the active_nh_list and also moving buckets from one nexthop to another. That should take care of all your concerns.
created_at: 2025-04-11T18:11:19Z
author_association: MEMBER
diff_hunk: @@ -267,8 +274,10 @@ def program_route_and_validate_fine_grained_ecmp(app_db, asic_db, state_db, ip_t
     ps = swsscommon.ProducerStateTable(app_db, ROUTE_TB)
     fvs = swsscommon.FieldValuePairs([("nexthop", ips), ("ifname", ifs)])
     ps.set(fg_nhg_prefix, fvs)
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                          fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    time.sleep(1)
path: tests/test_fgnhg.py
user: login: ashutosh-agrawal
body: Removed sleep.
created_at: 2025-04-11T18:17:31Z
author_association: MEMBER
diff_hunk: @@ -376,134 +391,158 @@ def fine_grained_ecmp_base_test(dvs, match_mode):
     dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03")
     dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
     dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+    time.sleep(1)
path: tests/test_fgnhg.py
user: login: ashutosh-agrawal
body: done
created_at: 2025-04-11T18:17:37Z
author_association: MEMBER
diff_hunk: @@ -376,134 +391,158 @@ def fine_grained_ecmp_base_test(dvs, match_mode):
     dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03")
     dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
     dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+    time.sleep(1)
     asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 5)
 
     asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
     nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
 
     nh_oid_map = get_nh_oid_map(asic_db)
+    memb_dict = {}
 
     ### Test scenarios with bank 0 having 0 members up and only bank 1 having members
     # ARP is not resolved for 10.0.0.7, so fg nhg should be created without 10.0.0.7
     nh_memb_exp_count = {"10.0.0.9":30,"10.0.0.11":30}
-    validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                            memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
     # Resolve ARP for 10.0.0.7
     asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
     dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    time.sleep(1)
path: tests/test_fgnhg.py
user: login: ashutosh-agrawal
body: done
created_at: 2025-04-11T18:17:45Z
author_association: MEMBER
diff_hunk: @@ -610,64 +651,112 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        auto active_nh_list = bank_member_change.active_nhs;
+        size_t act_nh_exp_size = exp_bucket_size;
 
+        if (num_nhs_with_one_more > 0)
+        {
+            act_nh_exp_size++;
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: done
created_at: 2025-04-11T18:18:01Z
author_association: MEMBER
diff_hunk: @@ -148,7 +162,7 @@ class FgNhgOrch : public Orch, public Observer
                     uint32_t bank, std::vector<BankMemberChanges> bank_member_changes,
                     std::map<NextHopKey,sai_object_id_t> &nhopgroup_members_set, const IpPrefix&);
     void calculateBankHashBucketStartIndices(FgNhgEntry *fgNhgEntry);
-    void setStateDbRouteEntry(const IpPrefix&, uint32_t index, NextHopKey nextHop);
+    void setStateDbRouteEntry(const IpPrefix&, uint32_t bucket_index, NextHopKey nextHop);
path: orchagent/fgnhgorch.h
user: login: ashutosh-agrawal
body: fixed.
created_at: 2025-04-11T18:18:12Z
author_association: MEMBER
diff_hunk: @@ -718,9 +818,16 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
                 }
                 else
                 {
-                    uint32_t last_elem = map_entry->at((*map_entry).size() - 1);
+                    if ((*map_entry).size() == exp_size_before_bkt_rem)
+                    {
+                        SWSS_LOG_INFO("%s Buckets size already at %d, don't remove more",
+                                it->to_string().c_str(), exp_size_before_bkt_rem);
+                        it = active_nh_list.erase(it);
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: I've rewritten this logic and this code fragment is no longer needed. 
created_at: 2025-04-11T18:18:58Z
author_association: MEMBER
diff_hunk: @@ -686,12 +775,21 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % total_nhs);
         uint32_t num_nhs_with_eq_to_exp = total_nhs - num_nhs_with_one_more;
         uint32_t add_nh_exp_bucket_size = exp_bucket_size;
+        uint32_t exp_size_before_bkt_rem = exp_bucket_size;
+
+        auto active_nh_list = bank_member_change.active_nhs;
+        auto compare_active_buckets = [&](NextHopKey e1, NextHopKey e2) {
+            return bank_fgnhg_map->at(e1).size() > bank_fgnhg_map->at(e2).size();
+        };
+        // arrange the active nhs in decreasing order of size of their hash bucket map
+        // so that we replace buckets of the active nhs which have higher bucket size first
+        std::sort(active_nh_list.begin(), active_nh_list.end(), compare_active_buckets);
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: Done.
created_at: 2025-04-11T18:19:07Z
author_association: MEMBER
diff_hunk: @@ -686,12 +775,21 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % total_nhs);
         uint32_t num_nhs_with_eq_to_exp = total_nhs - num_nhs_with_one_more;
         uint32_t add_nh_exp_bucket_size = exp_bucket_size;
+        uint32_t exp_size_before_bkt_rem = exp_bucket_size;
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: Done.
created_at: 2025-04-11T18:19:12Z
author_association: MEMBER
diff_hunk: @@ -204,29 +219,45 @@ void FgNhgOrch::setStateDbRouteEntry(const IpPrefix &ipPrefix, uint32_t index, N
     // Write to StateDb
     std::vector<FieldValueTuple> fvs;
 
-    // check if profile already exists - if yes - skip creation
+    // get the existing buckets from warm restart state table
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: Excellent suggestion. I've replaced the whole code snippet with a simple hset command.
created_at: 2025-04-15T17:53:00Z
author_association: MEMBER
diff_hunk: @@ -968,20 +1085,22 @@ bool FgNhgOrch::computeAndSetHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
                 (bank_member_changes[bank_idx].nhs_to_add.size() != 0 &&
                  bank_member_changes[bank_idx].nhs_to_del.size() != 0))
         {
+            SWSS_LOG_INFO("active nhs in bank %u", bank_idx);
             /* Active bank is is determined by there being active nhs on the bank OR
              * an edge case where all active_nhs went down(nhs_to_del > 0) BUT
-             * simultaneously, nhs were added(nhs_to_add > 0). 
+             * simultaneously, nhs were added(nhs_to_add > 0).
              * Route this to fn which deals with active banks
              */
-            if (!setActiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, 
-                        bank_idx, bank_idx, bank_member_changes, nhopgroup_members_set, ipPrefix))
+            if (!setActiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry,
+                        bank_idx, bank_member_changes[bank_idx], nhopgroup_members_set, ipPrefix))
             {
                 return false;
             }
         }
         else
         {
-            if (!setInactiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, 
+            SWSS_LOG_INFO("No active nhs in bank %u", bank_idx);
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: Removed.
created_at: 2025-04-15T17:54:21Z
author_association: MEMBER
diff_hunk: @@ -15,9 +15,10 @@
 #include <map>
 
 typedef uint32_t Bank;
+typedef uint32_t HashBucketIdx;
 typedef std::set<NextHopKey> ActiveNextHops;
-typedef std::vector<sai_object_id_t> FGNextHopGroupMembers;
-typedef std::vector<uint32_t> HashBuckets;
+typedef std::map<HashBucketIdx, sai_object_id_t> FGNextHopGroupMembers;
path: orchagent/fgnhgorch.h
user: login: ashutosh-agrawal
body: I've replaced map with a vector.
created_at: 2025-04-17T15:06:24Z
author_association: MEMBER
diff_hunk: @@ -968,20 +1085,22 @@ bool FgNhgOrch::computeAndSetHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
                 (bank_member_changes[bank_idx].nhs_to_add.size() != 0 &&
                  bank_member_changes[bank_idx].nhs_to_del.size() != 0))
         {
+            SWSS_LOG_INFO("active nhs in bank %u", bank_idx);
             /* Active bank is is determined by there being active nhs on the bank OR
              * an edge case where all active_nhs went down(nhs_to_del > 0) BUT
-             * simultaneously, nhs were added(nhs_to_add > 0). 
+             * simultaneously, nhs were added(nhs_to_add > 0).
              * Route this to fn which deals with active banks
              */
-            if (!setActiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, 
-                        bank_idx, bank_idx, bank_member_changes, nhopgroup_members_set, ipPrefix))
+            if (!setActiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry,
+                        bank_idx, bank_member_changes[bank_idx], nhopgroup_members_set, ipPrefix))
             {
                 return false;
             }
         }
         else
         {
-            if (!setInactiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, 
+            SWSS_LOG_INFO("No active nhs in bank %u", bank_idx);
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: Log isn't removed yet, can we remove it?
created_at: 2025-04-18T19:26:27Z
author_association: CONTRIBUTOR
diff_hunk: @@ -131,15 +138,22 @@ class FgNhgOrch : public Orch, public Observer
     // warm reboot support for recovery
     // < ip_prefix, < HashBuckets, nh_ip>>
     WarmBootRecoveryMap m_recoveryMap;
+    // < ip_prefix, < StateTblBucketMap>>
+    StateTblPrefixMap   m_stateTblPrefixMap;
path: orchagent/fgnhgorch.h
user: login: anish-n
body: remove
created_at: 2025-04-18T19:41:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -94,6 +98,9 @@ typedef struct
 
 typedef std::vector<string> NextHopIndexMap;
 typedef map<string, NextHopIndexMap> WarmBootRecoveryMap;
+typedef size_t state_tbl_idx;
+typedef map<HashBucketIdx, state_tbl_idx> StateTblBucketMap;
+typedef map<string, StateTblBucketMap> StateTblPrefixMap;
path: orchagent/fgnhgorch.h
user: login: anish-n
body: remove
created_at: 2025-04-18T19:42:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -593,63 +620,82 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        auto active_nh_list = bank_member_change.active_nhs;
+        bool move_bkt = false;
+        bool remove_nh = false;
 
-
-        while(del_idx < bank_member_change.nhs_to_del.size())
+        auto it = active_nh_list.begin();
+        while (del_idx < bank_member_change.nhs_to_del.size())
         {
             HashBuckets *hash_buckets = &(bank_fgnhg_map->at(bank_member_change.nhs_to_del[del_idx]));
-            for (uint32_t i = 0; i < hash_buckets->size(); i++)
-            {
-                NextHopKey round_robin_nh = bank_member_change.active_nhs[i %
-                    bank_member_change.active_nhs.size()];
 
-                if (!writeHashBucketChange(syncd_fg_route_entry, hash_buckets->at(i), 
-                        nhopgroup_members_set[round_robin_nh], ipPrefix, round_robin_nh))
+            uint32_t bkt_idx = 0;
+            while(bkt_idx < hash_buckets->size())
+            {
+                move_bkt = false; remove_nh = false;
+                // wraparound the active_nh_list if needed
+                if (it == active_nh_list.end())
                 {
-                    return false;
+                    it = active_nh_list.begin();
+                    if (it == active_nh_list.end())
+                    {
+                        // this can neven happen
+                        SWSS_LOG_ERROR("%s Unexpected no more active NHs before adding the %zu buckets, exp_bucket_size(%d)",
+                                       ipPrefix.to_string().c_str(),
+                                       hash_buckets->size(), exp_bucket_size);
+                        return false;
+                    }
                 }
-                bank_fgnhg_map->at(round_robin_nh).push_back(hash_buckets->at(i));
 
                 /* Logic below ensure that # hash buckets assigned to a nh is equalized,
-                 * we could have used simple round robin to reassign hash buckets to 
-                 * other available nhs, but for cases where # hash buckets is not 
+                 * we could have used simple round robin to reassign hash buckets to
+                 * other available nhs, but for cases where # hash buckets is not
                  * divisible by # of nhs, simple round robin can make the hash bucket
                  * distribution non-ideal, thereby nhs can attract unequal traffic */
-                if (num_nhs_with_one_more == 0)
+                if (bank_fgnhg_map->at(*it).size() == exp_bucket_size)
                 {
-                    if (bank_fgnhg_map->at(round_robin_nh).size() == exp_bucket_size)
+                    if (num_nhs_with_one_more == 0)
                     {
-                        SWSS_LOG_INFO("%s reached %d, don't remove more buckets", 
-                                (bank_member_change.active_nhs[i % bank_member_change.active_nhs.size()]).to_string().c_str(), 
-                                exp_bucket_size);
-                        bank_member_change.active_nhs.erase(bank_member_change.active_nhs.begin() + 
-                            (i % bank_member_change.active_nhs.size()));
+
+                        SWSS_LOG_INFO("%s already reached expected bucket size %d, don't add more buckets, bkt_idx(%d)",
+                                      (*it).to_string().c_str(), exp_bucket_size, bkt_idx);
+                        //it = active_nh_list.erase(it);
+                        remove_nh = true;
                     }
-                    else if (bank_fgnhg_map->at(round_robin_nh).size() > exp_bucket_size)
+                    else
                     {
-                        SWSS_LOG_WARN("Unexpected bucket size for nh %s, size %zu, exp_size %d",
-                                round_robin_nh.to_string().c_str(), bank_fgnhg_map->at(round_robin_nh).size(),
-                                exp_bucket_size);
+                        num_nhs_with_one_more--;
+                        SWSS_LOG_INFO("%s reached %d, don't add more buckets after this one, num_nhs_with_one_more remaining=%d, bkt_idx(%d)",
+                                      (*it).to_string().c_str(), exp_bucket_size + 1, num_nhs_with_one_more, bkt_idx);
+                        move_bkt = true;
+                        remove_nh = true;
                     }
                 }
                 else
                 {
-                    if (bank_fgnhg_map->at(round_robin_nh).size() == exp_bucket_size +1)
-                    {
+                    SWSS_LOG_INFO("%s has not reached min expected size of %d, keep adding to this nexthop, bkt_idx(%d)",
+                                  (*it).to_string().c_str(), exp_bucket_size, bkt_idx);
+                    move_bkt = true;
+                }
 
-                        SWSS_LOG_INFO("%s reached %d, don't remove more buckets num_nhs_with_one_more %d", 
-                                (bank_member_change.active_nhs[i %bank_member_change.active_nhs.size()]).to_string().c_str(), 
-                                exp_bucket_size +1, num_nhs_with_one_more -1);
-                        bank_member_change.active_nhs.erase(bank_member_change.active_nhs.begin() + 
-                            (i % bank_member_change.active_nhs.size()));
-                        num_nhs_with_one_more--;
-                    }
-                    else if (bank_fgnhg_map->at(round_robin_nh).size() > exp_bucket_size +1)
-                    {
-                        SWSS_LOG_WARN("Unexpected bucket size for nh %s, size %zu, exp_size %d",
-                                round_robin_nh.to_string().c_str(), bank_fgnhg_map->at(round_robin_nh).size(),
-                                exp_bucket_size + 1);
-                    }
+                if (move_bkt)
+                {
+                  if (!writeHashBucketChange(syncd_fg_route_entry, hash_buckets->at(bkt_idx),
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: fix indentation 
created_at: 2025-04-25T22:47:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -593,63 +620,82 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        auto active_nh_list = bank_member_change.active_nhs;
+        bool move_bkt = false;
+        bool remove_nh = false;
 
-
-        while(del_idx < bank_member_change.nhs_to_del.size())
+        auto it = active_nh_list.begin();
+        while (del_idx < bank_member_change.nhs_to_del.size())
         {
             HashBuckets *hash_buckets = &(bank_fgnhg_map->at(bank_member_change.nhs_to_del[del_idx]));
-            for (uint32_t i = 0; i < hash_buckets->size(); i++)
-            {
-                NextHopKey round_robin_nh = bank_member_change.active_nhs[i %
-                    bank_member_change.active_nhs.size()];
 
-                if (!writeHashBucketChange(syncd_fg_route_entry, hash_buckets->at(i), 
-                        nhopgroup_members_set[round_robin_nh], ipPrefix, round_robin_nh))
+            uint32_t bkt_idx = 0;
+            while(bkt_idx < hash_buckets->size())
+            {
+                move_bkt = false; remove_nh = false;
+                // wraparound the active_nh_list if needed
+                if (it == active_nh_list.end())
                 {
-                    return false;
+                    it = active_nh_list.begin();
+                    if (it == active_nh_list.end())
+                    {
+                        // this can neven happen
+                        SWSS_LOG_ERROR("%s Unexpected no more active NHs before adding the %zu buckets, exp_bucket_size(%d)",
+                                       ipPrefix.to_string().c_str(),
+                                       hash_buckets->size(), exp_bucket_size);
+                        return false;
+                    }
                 }
-                bank_fgnhg_map->at(round_robin_nh).push_back(hash_buckets->at(i));
 
                 /* Logic below ensure that # hash buckets assigned to a nh is equalized,
-                 * we could have used simple round robin to reassign hash buckets to 
-                 * other available nhs, but for cases where # hash buckets is not 
+                 * we could have used simple round robin to reassign hash buckets to
+                 * other available nhs, but for cases where # hash buckets is not
                  * divisible by # of nhs, simple round robin can make the hash bucket
                  * distribution non-ideal, thereby nhs can attract unequal traffic */
-                if (num_nhs_with_one_more == 0)
+                if (bank_fgnhg_map->at(*it).size() == exp_bucket_size)
                 {
-                    if (bank_fgnhg_map->at(round_robin_nh).size() == exp_bucket_size)
+                    if (num_nhs_with_one_more == 0)
                     {
-                        SWSS_LOG_INFO("%s reached %d, don't remove more buckets", 
-                                (bank_member_change.active_nhs[i % bank_member_change.active_nhs.size()]).to_string().c_str(), 
-                                exp_bucket_size);
-                        bank_member_change.active_nhs.erase(bank_member_change.active_nhs.begin() + 
-                            (i % bank_member_change.active_nhs.size()));
+
+                        SWSS_LOG_INFO("%s already reached expected bucket size %d, don't add more buckets, bkt_idx(%d)",
+                                      (*it).to_string().c_str(), exp_bucket_size, bkt_idx);
+                        //it = active_nh_list.erase(it);
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: remove comment
created_at: 2025-04-25T22:52:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -684,77 +735,83 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
                 add_nh_exp_bucket_size = exp_bucket_size + 1;
                 num_nhs_with_one_more--;
             }
-
-            while(bank_fgnhg_map->at(bank_member_change.nhs_to_add[add_idx]).size() != add_nh_exp_bucket_size)
+            // Print the values of total_nhs, exp_bucket_size, num_nhs_with_one_more, and add_nh_exp_bucket_size
+            SWSS_LOG_INFO("total_nhs: %u, exp_bucket_size: %u, num_nhs_with_one_more: %u, add_nh_exp_bucket_size: %u",
+                          total_nhs, exp_bucket_size, num_nhs_with_one_more, add_nh_exp_bucket_size);
+            for (const auto &nh : active_nh_list)
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: Added for debug and should be removed? 
created_at: 2025-04-26T18:31:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -684,77 +735,83 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
                 add_nh_exp_bucket_size = exp_bucket_size + 1;
                 num_nhs_with_one_more--;
             }
-
-            while(bank_fgnhg_map->at(bank_member_change.nhs_to_add[add_idx]).size() != add_nh_exp_bucket_size)
+            // Print the values of total_nhs, exp_bucket_size, num_nhs_with_one_more, and add_nh_exp_bucket_size
+            SWSS_LOG_INFO("total_nhs: %u, exp_bucket_size: %u, num_nhs_with_one_more: %u, add_nh_exp_bucket_size: %u",
+                          total_nhs, exp_bucket_size, num_nhs_with_one_more, add_nh_exp_bucket_size);
+            for (const auto &nh : active_nh_list)
             {
-                if (it == bank_member_change.active_nhs.end())
-                {
-                    it = bank_member_change.active_nhs.begin();
-                }
-                vector<uint32_t> *map_entry = &(bank_fgnhg_map->at(*it));
-                if ((*map_entry).size() <= 1)
+                SWSS_LOG_INFO("Active next-hop %s has %zu buckets assigned",
+                              nh.to_string().c_str(), bank_fgnhg_map->at(nh).size());
+            }
+            // add buckets to this nh until HashBuckets size has reached the expected size
+            while (bank_fgnhg_map->at(bank_member_change.nhs_to_add[add_idx]).size() != add_nh_exp_bucket_size)
+            {
+                move_bkt=false;
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: nit fix spacing move_bkt = false
created_at: 2025-04-26T18:32:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1640,6 +1940,19 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t)
             return false;
         }
 
+        // In the dynamic_nhg mode, only 1 FG_NHG_PREFIX is allowed for each FG_NHG. If ip_prefix 
+        // doesn't match with the existing prefixes in the fgNhgEntry, then return false.
+        if (fgNhg_entry->second.nhg_mode == DYNAMIC_FGNHG)
+        {
+            if (fgNhg_entry->second.prefixes.size() != 0)
path: orchagent/fgnhgorch.cpp
user: login: anish-n
body: The following is just a suggestion and more for future/a follow up PR if there is ever a need for it: there are 2 different structures in the code, FGNextHopGroupEntry for per prefix things via BGP/route entry, and FgNhgEntry for user defined fine grained nhg config.
 
Given the new use case where you need nexthops/links from the route entry instead of user config, my suggestion is to try adding a new pointer to a nexthops and links struct and storing it in the FGNextHopGroupEntry, and then refactoring to use nexthops/links from that route entry vs as the code is currently doing things from FgNhgEntry. That way you can support multiple prefixes per FG_NHG(which sounds like the ideal design imho).
created_at: 2025-04-26T19:45:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -684,77 +735,83 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
                 add_nh_exp_bucket_size = exp_bucket_size + 1;
                 num_nhs_with_one_more--;
             }
-
-            while(bank_fgnhg_map->at(bank_member_change.nhs_to_add[add_idx]).size() != add_nh_exp_bucket_size)
+            // Print the values of total_nhs, exp_bucket_size, num_nhs_with_one_more, and add_nh_exp_bucket_size
+            SWSS_LOG_INFO("total_nhs: %u, exp_bucket_size: %u, num_nhs_with_one_more: %u, add_nh_exp_bucket_size: %u",
+                          total_nhs, exp_bucket_size, num_nhs_with_one_more, add_nh_exp_bucket_size);
+            for (const auto &nh : active_nh_list)
             {
-                if (it == bank_member_change.active_nhs.end())
-                {
-                    it = bank_member_change.active_nhs.begin();
-                }
-                vector<uint32_t> *map_entry = &(bank_fgnhg_map->at(*it));
-                if ((*map_entry).size() <= 1)
+                SWSS_LOG_INFO("Active next-hop %s has %zu buckets assigned",
+                              nh.to_string().c_str(), bank_fgnhg_map->at(nh).size());
+            }
+            // add buckets to this nh until HashBuckets size has reached the expected size
+            while (bank_fgnhg_map->at(bank_member_change.nhs_to_add[add_idx]).size() != add_nh_exp_bucket_size)
+            {
+                move_bkt=false;
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: Fixed.
created_at: 2025-04-28T17:09:41Z
author_association: MEMBER
diff_hunk: @@ -593,63 +620,82 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        auto active_nh_list = bank_member_change.active_nhs;
+        bool move_bkt = false;
+        bool remove_nh = false;
 
-
-        while(del_idx < bank_member_change.nhs_to_del.size())
+        auto it = active_nh_list.begin();
+        while (del_idx < bank_member_change.nhs_to_del.size())
         {
             HashBuckets *hash_buckets = &(bank_fgnhg_map->at(bank_member_change.nhs_to_del[del_idx]));
-            for (uint32_t i = 0; i < hash_buckets->size(); i++)
-            {
-                NextHopKey round_robin_nh = bank_member_change.active_nhs[i %
-                    bank_member_change.active_nhs.size()];
 
-                if (!writeHashBucketChange(syncd_fg_route_entry, hash_buckets->at(i), 
-                        nhopgroup_members_set[round_robin_nh], ipPrefix, round_robin_nh))
+            uint32_t bkt_idx = 0;
+            while(bkt_idx < hash_buckets->size())
+            {
+                move_bkt = false; remove_nh = false;
+                // wraparound the active_nh_list if needed
+                if (it == active_nh_list.end())
                 {
-                    return false;
+                    it = active_nh_list.begin();
+                    if (it == active_nh_list.end())
+                    {
+                        // this can neven happen
+                        SWSS_LOG_ERROR("%s Unexpected no more active NHs before adding the %zu buckets, exp_bucket_size(%d)",
+                                       ipPrefix.to_string().c_str(),
+                                       hash_buckets->size(), exp_bucket_size);
+                        return false;
+                    }
                 }
-                bank_fgnhg_map->at(round_robin_nh).push_back(hash_buckets->at(i));
 
                 /* Logic below ensure that # hash buckets assigned to a nh is equalized,
-                 * we could have used simple round robin to reassign hash buckets to 
-                 * other available nhs, but for cases where # hash buckets is not 
+                 * we could have used simple round robin to reassign hash buckets to
+                 * other available nhs, but for cases where # hash buckets is not
                  * divisible by # of nhs, simple round robin can make the hash bucket
                  * distribution non-ideal, thereby nhs can attract unequal traffic */
-                if (num_nhs_with_one_more == 0)
+                if (bank_fgnhg_map->at(*it).size() == exp_bucket_size)
                 {
-                    if (bank_fgnhg_map->at(round_robin_nh).size() == exp_bucket_size)
+                    if (num_nhs_with_one_more == 0)
                     {
-                        SWSS_LOG_INFO("%s reached %d, don't remove more buckets", 
-                                (bank_member_change.active_nhs[i % bank_member_change.active_nhs.size()]).to_string().c_str(), 
-                                exp_bucket_size);
-                        bank_member_change.active_nhs.erase(bank_member_change.active_nhs.begin() + 
-                            (i % bank_member_change.active_nhs.size()));
+
+                        SWSS_LOG_INFO("%s already reached expected bucket size %d, don't add more buckets, bkt_idx(%d)",
+                                      (*it).to_string().c_str(), exp_bucket_size, bkt_idx);
+                        //it = active_nh_list.erase(it);
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: Removed.

created_at: 2025-04-28T17:09:55Z
author_association: MEMBER
diff_hunk: @@ -593,63 +620,82 @@ bool FgNhgOrch::setActiveBankHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
             fgNhgEntry->hash_bucket_indices[syncd_bank].start_index;
         uint32_t exp_bucket_size = num_buckets_in_bank / (uint32_t)bank_member_change.active_nhs.size();
         uint32_t num_nhs_with_one_more = (num_buckets_in_bank % (uint32_t)bank_member_change.active_nhs.size());
+        auto active_nh_list = bank_member_change.active_nhs;
+        bool move_bkt = false;
+        bool remove_nh = false;
 
-
-        while(del_idx < bank_member_change.nhs_to_del.size())
+        auto it = active_nh_list.begin();
+        while (del_idx < bank_member_change.nhs_to_del.size())
         {
             HashBuckets *hash_buckets = &(bank_fgnhg_map->at(bank_member_change.nhs_to_del[del_idx]));
-            for (uint32_t i = 0; i < hash_buckets->size(); i++)
-            {
-                NextHopKey round_robin_nh = bank_member_change.active_nhs[i %
-                    bank_member_change.active_nhs.size()];
 
-                if (!writeHashBucketChange(syncd_fg_route_entry, hash_buckets->at(i), 
-                        nhopgroup_members_set[round_robin_nh], ipPrefix, round_robin_nh))
+            uint32_t bkt_idx = 0;
+            while(bkt_idx < hash_buckets->size())
+            {
+                move_bkt = false; remove_nh = false;
+                // wraparound the active_nh_list if needed
+                if (it == active_nh_list.end())
                 {
-                    return false;
+                    it = active_nh_list.begin();
+                    if (it == active_nh_list.end())
+                    {
+                        // this can neven happen
+                        SWSS_LOG_ERROR("%s Unexpected no more active NHs before adding the %zu buckets, exp_bucket_size(%d)",
+                                       ipPrefix.to_string().c_str(),
+                                       hash_buckets->size(), exp_bucket_size);
+                        return false;
+                    }
                 }
-                bank_fgnhg_map->at(round_robin_nh).push_back(hash_buckets->at(i));
 
                 /* Logic below ensure that # hash buckets assigned to a nh is equalized,
-                 * we could have used simple round robin to reassign hash buckets to 
-                 * other available nhs, but for cases where # hash buckets is not 
+                 * we could have used simple round robin to reassign hash buckets to
+                 * other available nhs, but for cases where # hash buckets is not
                  * divisible by # of nhs, simple round robin can make the hash bucket
                  * distribution non-ideal, thereby nhs can attract unequal traffic */
-                if (num_nhs_with_one_more == 0)
+                if (bank_fgnhg_map->at(*it).size() == exp_bucket_size)
                 {
-                    if (bank_fgnhg_map->at(round_robin_nh).size() == exp_bucket_size)
+                    if (num_nhs_with_one_more == 0)
                     {
-                        SWSS_LOG_INFO("%s reached %d, don't remove more buckets", 
-                                (bank_member_change.active_nhs[i % bank_member_change.active_nhs.size()]).to_string().c_str(), 
-                                exp_bucket_size);
-                        bank_member_change.active_nhs.erase(bank_member_change.active_nhs.begin() + 
-                            (i % bank_member_change.active_nhs.size()));
+
+                        SWSS_LOG_INFO("%s already reached expected bucket size %d, don't add more buckets, bkt_idx(%d)",
+                                      (*it).to_string().c_str(), exp_bucket_size, bkt_idx);
+                        //it = active_nh_list.erase(it);
+                        remove_nh = true;
                     }
-                    else if (bank_fgnhg_map->at(round_robin_nh).size() > exp_bucket_size)
+                    else
                     {
-                        SWSS_LOG_WARN("Unexpected bucket size for nh %s, size %zu, exp_size %d",
-                                round_robin_nh.to_string().c_str(), bank_fgnhg_map->at(round_robin_nh).size(),
-                                exp_bucket_size);
+                        num_nhs_with_one_more--;
+                        SWSS_LOG_INFO("%s reached %d, don't add more buckets after this one, num_nhs_with_one_more remaining=%d, bkt_idx(%d)",
+                                      (*it).to_string().c_str(), exp_bucket_size + 1, num_nhs_with_one_more, bkt_idx);
+                        move_bkt = true;
+                        remove_nh = true;
                     }
                 }
                 else
                 {
-                    if (bank_fgnhg_map->at(round_robin_nh).size() == exp_bucket_size +1)
-                    {
+                    SWSS_LOG_INFO("%s has not reached min expected size of %d, keep adding to this nexthop, bkt_idx(%d)",
+                                  (*it).to_string().c_str(), exp_bucket_size, bkt_idx);
+                    move_bkt = true;
+                }
 
-                        SWSS_LOG_INFO("%s reached %d, don't remove more buckets num_nhs_with_one_more %d", 
-                                (bank_member_change.active_nhs[i %bank_member_change.active_nhs.size()]).to_string().c_str(), 
-                                exp_bucket_size +1, num_nhs_with_one_more -1);
-                        bank_member_change.active_nhs.erase(bank_member_change.active_nhs.begin() + 
-                            (i % bank_member_change.active_nhs.size()));
-                        num_nhs_with_one_more--;
-                    }
-                    else if (bank_fgnhg_map->at(round_robin_nh).size() > exp_bucket_size +1)
-                    {
-                        SWSS_LOG_WARN("Unexpected bucket size for nh %s, size %zu, exp_size %d",
-                                round_robin_nh.to_string().c_str(), bank_fgnhg_map->at(round_robin_nh).size(),
-                                exp_bucket_size + 1);
-                    }
+                if (move_bkt)
+                {
+                  if (!writeHashBucketChange(syncd_fg_route_entry, hash_buckets->at(bkt_idx),
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: Fixed.
created_at: 2025-04-28T17:10:01Z
author_association: MEMBER
diff_hunk: @@ -94,6 +98,9 @@ typedef struct
 
 typedef std::vector<string> NextHopIndexMap;
 typedef map<string, NextHopIndexMap> WarmBootRecoveryMap;
+typedef size_t state_tbl_idx;
+typedef map<HashBucketIdx, state_tbl_idx> StateTblBucketMap;
+typedef map<string, StateTblBucketMap> StateTblPrefixMap;
path: orchagent/fgnhgorch.h
user: login: ashutosh-agrawal
body: Removed
created_at: 2025-04-28T17:10:08Z
author_association: MEMBER
diff_hunk: @@ -131,15 +138,22 @@ class FgNhgOrch : public Orch, public Observer
     // warm reboot support for recovery
     // < ip_prefix, < HashBuckets, nh_ip>>
     WarmBootRecoveryMap m_recoveryMap;
+    // < ip_prefix, < StateTblBucketMap>>
+    StateTblPrefixMap   m_stateTblPrefixMap;
path: orchagent/fgnhgorch.h
user: login: ashutosh-agrawal
body: Removed
created_at: 2025-04-28T17:10:15Z
author_association: MEMBER
diff_hunk: @@ -968,20 +1085,22 @@ bool FgNhgOrch::computeAndSetHashBucketChanges(FGNextHopGroupEntry *syncd_fg_rou
                 (bank_member_changes[bank_idx].nhs_to_add.size() != 0 &&
                  bank_member_changes[bank_idx].nhs_to_del.size() != 0))
         {
+            SWSS_LOG_INFO("active nhs in bank %u", bank_idx);
             /* Active bank is is determined by there being active nhs on the bank OR
              * an edge case where all active_nhs went down(nhs_to_del > 0) BUT
-             * simultaneously, nhs were added(nhs_to_add > 0). 
+             * simultaneously, nhs were added(nhs_to_add > 0).
              * Route this to fn which deals with active banks
              */
-            if (!setActiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, 
-                        bank_idx, bank_idx, bank_member_changes, nhopgroup_members_set, ipPrefix))
+            if (!setActiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry,
+                        bank_idx, bank_member_changes[bank_idx], nhopgroup_members_set, ipPrefix))
             {
                 return false;
             }
         }
         else
         {
-            if (!setInactiveBankHashBucketChanges(syncd_fg_route_entry, fgNhgEntry, 
+            SWSS_LOG_INFO("No active nhs in bank %u", bank_idx);
path: orchagent/fgnhgorch.cpp
user: login: ashutosh-agrawal
body: Sorry, removed it now.

created_at: 2025-04-28T17:16:30Z
author_association: MEMBER
diff_hunk: @@ -242,17 +246,52 @@
     for idx,memb in memb_dict.items():
         assert memb == 0
 
-def validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+def verify_fg_state_db_for_even_distribution(state_db, fg_nhg_prefix, bucket_size, nh_ip_count):
+    def _access_function():
+        false_ret = (False, '')
+        ret = True
+        keys = state_db.get_keys("FG_ROUTE_TABLE")
+        if not keys:
+            return false_ret
+        for key in keys:
+            if key != fg_nhg_prefix:
+                continue
+            fvs = state_db.get_entry("FG_ROUTE_TABLE", key)
+            if not fvs:
+                return false_ret
+            member_count = {}
+            for key, value in fvs.items():
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Nested loops with same variable

Nested for statement uses loop variable 'key' of enclosing [for statement](1).

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/616)
created_at: 2025-04-29T17:06:38Z
author_association: NONE
diff_hunk: @@ -317,6 +373,12 @@
     bucket_size = 60
     ip_to_if_map = {}
 
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Unused local variable

Variable old_log_level is not used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/605)
created_at: 2025-04-29T17:06:38Z
author_association: NONE
diff_hunk: @@ -589,10 +679,12 @@
 
         nh_oid_map = {}
         nh_oid_map = get_nh_oid_map(asic_db)
+        memb_dict = {}
 
         nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
-        validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                                fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 60
+        memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Unused local variable

Variable memb_dict is not used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/606)
created_at: 2025-04-29T17:06:38Z
author_association: NONE
diff_hunk: @@ -624,6 +716,624 @@
         remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
+def fine_grained_ecmp_match_mode_prefix_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 6
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 60
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Unused local variable

Variable old_log_level is not used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/607)
created_at: 2025-04-29T17:06:39Z
author_association: NONE
diff_hunk: @@ -624,6 +716,624 @@
         remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
+def fine_grained_ecmp_match_mode_prefix_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 6
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 60
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for 3 next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    # Add route with 3 next-hops
+    print("Add route with 3 next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members
+    print("Test scenarios with 3 members")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                       memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 next hop
+    print("FGNHG Bring down 1 next hop")
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # (Bring down 2 next hop and bring up 1 next hop)
+    print("Bring down 2 next hop and bring up 1 next hop")
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 60
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 next hops
+    print("Bring up 2 next hops")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 3 more next-hops
+    print("Bring up 3 more next-hops")
+    # First Resolve ARP for 3 more next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:08")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:09")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    # Test warm reboot
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 3 next-hops
+    print("Bring down 3 next-hops")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.5":20,"10.0.0.11":20}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 member and bring up 1 member at the same time
+    print("Bring down 1 member and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 2 members and bring up 1 member at the same time
+    print("Bring down 2 members and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 members and bring down 1 member at the same time
+    print("Bring up 2 members and bring down 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bringup all the inactive nexthops at the same time
+    print("Bringup all the inactive nexthops at the same time")
+    num_exp_changes = 30
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Variable defined multiple times

This assignment to 'memb_dict' is unnecessary as it is [redefined](1) before this value is used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/602)
created_at: 2025-04-29T17:06:39Z
author_association: NONE
diff_hunk: @@ -624,6 +716,624 @@
         remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
+def fine_grained_ecmp_match_mode_prefix_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 6
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 60
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for 3 next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    # Add route with 3 next-hops
+    print("Add route with 3 next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members
+    print("Test scenarios with 3 members")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                       memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 next hop
+    print("FGNHG Bring down 1 next hop")
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # (Bring down 2 next hop and bring up 1 next hop)
+    print("Bring down 2 next hop and bring up 1 next hop")
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 60
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 next hops
+    print("Bring up 2 next hops")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 3 more next-hops
+    print("Bring up 3 more next-hops")
+    # First Resolve ARP for 3 more next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:08")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:09")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    # Test warm reboot
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 3 next-hops
+    print("Bring down 3 next-hops")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.5":20,"10.0.0.11":20}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 member and bring up 1 member at the same time
+    print("Bring down 1 member and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 2 members and bring up 1 member at the same time
+    print("Bring down 2 members and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 members and bring down 1 member at the same time
+    print("Bring up 2 members and bring down 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bringup all the inactive nexthops at the same time
+    print("Bringup all the inactive nexthops at the same time")
+    num_exp_changes = 30
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+     # Remove route
+    print("Remove route")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+
+    # validate routes and nhg member in asic db, route entry in state db are removed
+    print("validate routes and nhg member in asic db, route entry in state db are removed")
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    # Remove fgnhg prefix and group
+    print("Remove fgnhg prefix")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+    # Nothing we can wait for in terms of db entries, we sleep here
+    # to give the sw enough time to delete the entry
+    time.sleep(1)
+
+    # Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see
+    # standard(non-Fine grained) ECMP behavior
+    print("Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see")
+    print("standard(non-Fine grained) ECMP behavior")
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+            ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+    validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3)
+
+    # add back fgnhg group and prefix: The regular route should transition to fine grained ECMP
+    print("add fgnhg group and prefix: The regular route should transition to fine grained ECMP")
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    # Validate the transistion to Fine Grained ECMP
+    print("Validate the transistion to Fine Grained ECMP")
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = {}
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Unused local variable

Variable memb_dict is not used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/608)
created_at: 2025-04-29T17:06:39Z
author_association: NONE
diff_hunk: @@ -624,6 +716,624 @@
         remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
+def fine_grained_ecmp_match_mode_prefix_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 6
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 60
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for 3 next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    # Add route with 3 next-hops
+    print("Add route with 3 next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members
+    print("Test scenarios with 3 members")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                       memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 next hop
+    print("FGNHG Bring down 1 next hop")
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # (Bring down 2 next hop and bring up 1 next hop)
+    print("Bring down 2 next hop and bring up 1 next hop")
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 60
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 next hops
+    print("Bring up 2 next hops")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 3 more next-hops
+    print("Bring up 3 more next-hops")
+    # First Resolve ARP for 3 more next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:08")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:09")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    # Test warm reboot
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 3 next-hops
+    print("Bring down 3 next-hops")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.5":20,"10.0.0.11":20}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 member and bring up 1 member at the same time
+    print("Bring down 1 member and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 2 members and bring up 1 member at the same time
+    print("Bring down 2 members and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 members and bring down 1 member at the same time
+    print("Bring up 2 members and bring down 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bringup all the inactive nexthops at the same time
+    print("Bringup all the inactive nexthops at the same time")
+    num_exp_changes = 30
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+     # Remove route
+    print("Remove route")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+
+    # validate routes and nhg member in asic db, route entry in state db are removed
+    print("validate routes and nhg member in asic db, route entry in state db are removed")
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    # Remove fgnhg prefix and group
+    print("Remove fgnhg prefix")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+    # Nothing we can wait for in terms of db entries, we sleep here
+    # to give the sw enough time to delete the entry
+    time.sleep(1)
+
+    # Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see
+    # standard(non-Fine grained) ECMP behavior
+    print("Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see")
+    print("standard(non-Fine grained) ECMP behavior")
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+            ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+    validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3)
+
+    # add back fgnhg group and prefix: The regular route should transition to fine grained ECMP
+    print("add fgnhg group and prefix: The regular route should transition to fine grained ECMP")
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    # Validate the transistion to Fine Grained ECMP
+    print("Validate the transistion to Fine Grained ECMP")
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = {}
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # remove prefix entry
+    print("remove prefix entry")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+
+    # Cleanup all FG, arp and interface
+    print("Cleanup all FG, arp and interface")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, IF_TB, ip_pref_key)
+        remove_entry(config_db, IF_TB, if_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_multi_route_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM0_NHs = 6
+    NUM1_NHs = 4
+    fg_nhg_name0 = "fgnhg0_v4"
+    fg_nhg_name1 = "fgnhg1_v4"
+    fg_nhg_prefix0 = "2.2.2.0/24"
+    fg_nhg_prefix1 = "3.3.3.0/24"
+    bucket0_size = 60
+    bucket1_size = 24
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Unused local variable

Variable old_log_level is not used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/609)
created_at: 2025-04-29T17:06:39Z
author_association: NONE
diff_hunk: @@ -624,6 +716,624 @@
         remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
+def fine_grained_ecmp_match_mode_prefix_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 6
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 60
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for 3 next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    # Add route with 3 next-hops
+    print("Add route with 3 next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members
+    print("Test scenarios with 3 members")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                       memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 next hop
+    print("FGNHG Bring down 1 next hop")
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # (Bring down 2 next hop and bring up 1 next hop)
+    print("Bring down 2 next hop and bring up 1 next hop")
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 60
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 next hops
+    print("Bring up 2 next hops")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 3 more next-hops
+    print("Bring up 3 more next-hops")
+    # First Resolve ARP for 3 more next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:08")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:09")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    # Test warm reboot
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 3 next-hops
+    print("Bring down 3 next-hops")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.5":20,"10.0.0.11":20}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 member and bring up 1 member at the same time
+    print("Bring down 1 member and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 2 members and bring up 1 member at the same time
+    print("Bring down 2 members and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 members and bring down 1 member at the same time
+    print("Bring up 2 members and bring down 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bringup all the inactive nexthops at the same time
+    print("Bringup all the inactive nexthops at the same time")
+    num_exp_changes = 30
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+     # Remove route
+    print("Remove route")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+
+    # validate routes and nhg member in asic db, route entry in state db are removed
+    print("validate routes and nhg member in asic db, route entry in state db are removed")
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    # Remove fgnhg prefix and group
+    print("Remove fgnhg prefix")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+    # Nothing we can wait for in terms of db entries, we sleep here
+    # to give the sw enough time to delete the entry
+    time.sleep(1)
+
+    # Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see
+    # standard(non-Fine grained) ECMP behavior
+    print("Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see")
+    print("standard(non-Fine grained) ECMP behavior")
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+            ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+    validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3)
+
+    # add back fgnhg group and prefix: The regular route should transition to fine grained ECMP
+    print("add fgnhg group and prefix: The regular route should transition to fine grained ECMP")
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    # Validate the transistion to Fine Grained ECMP
+    print("Validate the transistion to Fine Grained ECMP")
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = {}
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # remove prefix entry
+    print("remove prefix entry")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+
+    # Cleanup all FG, arp and interface
+    print("Cleanup all FG, arp and interface")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, IF_TB, ip_pref_key)
+        remove_entry(config_db, IF_TB, if_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_multi_route_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM0_NHs = 6
+    NUM1_NHs = 4
+    fg_nhg_name0 = "fgnhg0_v4"
+    fg_nhg_name1 = "fgnhg1_v4"
+    fg_nhg_prefix0 = "2.2.2.0/24"
+    fg_nhg_prefix1 = "3.3.3.0/24"
+    bucket0_size = 60
+    bucket1_size = 24
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    ### Create first fine grained next hop group and prefix
+    fvs = {"bucket_size": str(bucket0_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM0_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name0, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name0}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix0, fvs)
+
+    ### Create shared interfaces (Last two Nexthops for the 1st prefix are shared with the 2nd prefix)
+    for i in range(0,NUM0_NHs+NUM1_NHs-2):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        fvs = {"vlanid": str((i+1)*4)}
+        create_entry(config_db, VLAN_TB, vlan_name_key, fvs)
+        fvs = {"tagging_mode": "untagged"}
+        create_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key, fvs)
+        create_entry(config_db, VLAN_IF_TB, vlan_name_key, fvs_nul)
+        create_entry(config_db, VLAN_IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = vlan_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    ### Create Route for the first Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan16,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix0, fvs)
+    # No ASIC_DB entry we can wait for since ARP is not resolved yet,
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix0:
+            found_route = True
+            break
+
+    # Since we didn't populate ARP yet, route should point to RIF for kernel arp resolution to occur
+    assert (found_route == True)
+    validate_asic_nhg_router_interface(asic_db, fg_nhg_prefix0)
+
+    # Add ARP entries for both prefixes
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03")
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+    dvs.runcmd("arp -s 10.0.0.13 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.15 00:00:00:00:00:08")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 8)
+
+    # verify correct number of nhg members were created for the first prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size)
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Variable defined multiple times

This assignment to 'nh_oid_map' is unnecessary as it is [redefined](1) before this value is used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/603)
created_at: 2025-04-29T17:06:39Z
author_association: NONE
diff_hunk: @@ -624,6 +716,624 @@
         remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
+def fine_grained_ecmp_match_mode_prefix_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 6
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 60
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for 3 next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    # Add route with 3 next-hops
+    print("Add route with 3 next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members
+    print("Test scenarios with 3 members")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                       memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 next hop
+    print("FGNHG Bring down 1 next hop")
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # (Bring down 2 next hop and bring up 1 next hop)
+    print("Bring down 2 next hop and bring up 1 next hop")
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 60
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 next hops
+    print("Bring up 2 next hops")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 3 more next-hops
+    print("Bring up 3 more next-hops")
+    # First Resolve ARP for 3 more next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:08")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:09")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    # Test warm reboot
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 3 next-hops
+    print("Bring down 3 next-hops")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.5":20,"10.0.0.11":20}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 member and bring up 1 member at the same time
+    print("Bring down 1 member and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 2 members and bring up 1 member at the same time
+    print("Bring down 2 members and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 members and bring down 1 member at the same time
+    print("Bring up 2 members and bring down 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bringup all the inactive nexthops at the same time
+    print("Bringup all the inactive nexthops at the same time")
+    num_exp_changes = 30
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+     # Remove route
+    print("Remove route")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+
+    # validate routes and nhg member in asic db, route entry in state db are removed
+    print("validate routes and nhg member in asic db, route entry in state db are removed")
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    # Remove fgnhg prefix and group
+    print("Remove fgnhg prefix")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+    # Nothing we can wait for in terms of db entries, we sleep here
+    # to give the sw enough time to delete the entry
+    time.sleep(1)
+
+    # Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see
+    # standard(non-Fine grained) ECMP behavior
+    print("Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see")
+    print("standard(non-Fine grained) ECMP behavior")
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+            ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+    validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3)
+
+    # add back fgnhg group and prefix: The regular route should transition to fine grained ECMP
+    print("add fgnhg group and prefix: The regular route should transition to fine grained ECMP")
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    # Validate the transistion to Fine Grained ECMP
+    print("Validate the transistion to Fine Grained ECMP")
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = {}
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # remove prefix entry
+    print("remove prefix entry")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+
+    # Cleanup all FG, arp and interface
+    print("Cleanup all FG, arp and interface")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, IF_TB, ip_pref_key)
+        remove_entry(config_db, IF_TB, if_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_multi_route_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM0_NHs = 6
+    NUM1_NHs = 4
+    fg_nhg_name0 = "fgnhg0_v4"
+    fg_nhg_name1 = "fgnhg1_v4"
+    fg_nhg_prefix0 = "2.2.2.0/24"
+    fg_nhg_prefix1 = "3.3.3.0/24"
+    bucket0_size = 60
+    bucket1_size = 24
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    ### Create first fine grained next hop group and prefix
+    fvs = {"bucket_size": str(bucket0_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM0_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name0, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name0}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix0, fvs)
+
+    ### Create shared interfaces (Last two Nexthops for the 1st prefix are shared with the 2nd prefix)
+    for i in range(0,NUM0_NHs+NUM1_NHs-2):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        fvs = {"vlanid": str((i+1)*4)}
+        create_entry(config_db, VLAN_TB, vlan_name_key, fvs)
+        fvs = {"tagging_mode": "untagged"}
+        create_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key, fvs)
+        create_entry(config_db, VLAN_IF_TB, vlan_name_key, fvs_nul)
+        create_entry(config_db, VLAN_IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = vlan_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    ### Create Route for the first Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan16,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix0, fvs)
+    # No ASIC_DB entry we can wait for since ARP is not resolved yet,
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix0:
+            found_route = True
+            break
+
+    # Since we didn't populate ARP yet, route should point to RIF for kernel arp resolution to occur
+    assert (found_route == True)
+    validate_asic_nhg_router_interface(asic_db, fg_nhg_prefix0)
+
+    # Add ARP entries for both prefixes
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03")
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+    dvs.runcmd("arp -s 10.0.0.13 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.15 00:00:00:00:00:08")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 8)
+
+    # verify correct number of nhg members were created for the first prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size)
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Create 2nd next hop group and prefix
+    fvs = {"bucket_size": str(bucket1_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM1_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name1, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name1}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix1, fvs)
+
+    # Create Route for the 2nd Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.13,10.0.0.15,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan28,Vlan32,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix1, fvs)
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix1:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    # verify correct number of nhg members were created for the 2nd prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members for the first prefix and 4 members for the 2nd prefix
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict0 = {}
+    num_exp_changes = 60
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    memb_dict1 = {}
+    num_exp_changes = 24
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Test warm reboot
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    num_exp_changes = 0
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 1 common next hop
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":8,"10.0.0.15":8,"10.0.0.11":8}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring it back up
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 3 next-hops
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 40
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":12,"10.0.0.11":12}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bringup all the inactive nexthops at the same time
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 50
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Unused local variable

Variable memb_dict0 is not used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/610)
created_at: 2025-04-29T17:06:39Z
author_association: NONE
diff_hunk: @@ -624,6 +716,624 @@
         remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
+def fine_grained_ecmp_match_mode_prefix_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 6
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 60
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for 3 next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    # Add route with 3 next-hops
+    print("Add route with 3 next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members
+    print("Test scenarios with 3 members")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                       memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 next hop
+    print("FGNHG Bring down 1 next hop")
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # (Bring down 2 next hop and bring up 1 next hop)
+    print("Bring down 2 next hop and bring up 1 next hop")
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 60
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 next hops
+    print("Bring up 2 next hops")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 3 more next-hops
+    print("Bring up 3 more next-hops")
+    # First Resolve ARP for 3 more next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:08")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:09")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    # Test warm reboot
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 3 next-hops
+    print("Bring down 3 next-hops")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.5":20,"10.0.0.11":20}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 member and bring up 1 member at the same time
+    print("Bring down 1 member and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 2 members and bring up 1 member at the same time
+    print("Bring down 2 members and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 members and bring down 1 member at the same time
+    print("Bring up 2 members and bring down 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bringup all the inactive nexthops at the same time
+    print("Bringup all the inactive nexthops at the same time")
+    num_exp_changes = 30
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+     # Remove route
+    print("Remove route")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+
+    # validate routes and nhg member in asic db, route entry in state db are removed
+    print("validate routes and nhg member in asic db, route entry in state db are removed")
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    # Remove fgnhg prefix and group
+    print("Remove fgnhg prefix")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+    # Nothing we can wait for in terms of db entries, we sleep here
+    # to give the sw enough time to delete the entry
+    time.sleep(1)
+
+    # Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see
+    # standard(non-Fine grained) ECMP behavior
+    print("Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see")
+    print("standard(non-Fine grained) ECMP behavior")
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+            ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+    validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3)
+
+    # add back fgnhg group and prefix: The regular route should transition to fine grained ECMP
+    print("add fgnhg group and prefix: The regular route should transition to fine grained ECMP")
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    # Validate the transistion to Fine Grained ECMP
+    print("Validate the transistion to Fine Grained ECMP")
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = {}
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # remove prefix entry
+    print("remove prefix entry")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+
+    # Cleanup all FG, arp and interface
+    print("Cleanup all FG, arp and interface")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, IF_TB, ip_pref_key)
+        remove_entry(config_db, IF_TB, if_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_multi_route_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM0_NHs = 6
+    NUM1_NHs = 4
+    fg_nhg_name0 = "fgnhg0_v4"
+    fg_nhg_name1 = "fgnhg1_v4"
+    fg_nhg_prefix0 = "2.2.2.0/24"
+    fg_nhg_prefix1 = "3.3.3.0/24"
+    bucket0_size = 60
+    bucket1_size = 24
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    ### Create first fine grained next hop group and prefix
+    fvs = {"bucket_size": str(bucket0_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM0_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name0, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name0}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix0, fvs)
+
+    ### Create shared interfaces (Last two Nexthops for the 1st prefix are shared with the 2nd prefix)
+    for i in range(0,NUM0_NHs+NUM1_NHs-2):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        fvs = {"vlanid": str((i+1)*4)}
+        create_entry(config_db, VLAN_TB, vlan_name_key, fvs)
+        fvs = {"tagging_mode": "untagged"}
+        create_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key, fvs)
+        create_entry(config_db, VLAN_IF_TB, vlan_name_key, fvs_nul)
+        create_entry(config_db, VLAN_IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = vlan_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    ### Create Route for the first Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan16,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix0, fvs)
+    # No ASIC_DB entry we can wait for since ARP is not resolved yet,
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix0:
+            found_route = True
+            break
+
+    # Since we didn't populate ARP yet, route should point to RIF for kernel arp resolution to occur
+    assert (found_route == True)
+    validate_asic_nhg_router_interface(asic_db, fg_nhg_prefix0)
+
+    # Add ARP entries for both prefixes
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03")
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+    dvs.runcmd("arp -s 10.0.0.13 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.15 00:00:00:00:00:08")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 8)
+
+    # verify correct number of nhg members were created for the first prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size)
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Create 2nd next hop group and prefix
+    fvs = {"bucket_size": str(bucket1_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM1_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name1, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name1}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix1, fvs)
+
+    # Create Route for the 2nd Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.13,10.0.0.15,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan28,Vlan32,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix1, fvs)
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix1:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    # verify correct number of nhg members were created for the 2nd prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members for the first prefix and 4 members for the 2nd prefix
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict0 = {}
+    num_exp_changes = 60
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    memb_dict1 = {}
+    num_exp_changes = 24
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Test warm reboot
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    num_exp_changes = 0
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 1 common next hop
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":8,"10.0.0.15":8,"10.0.0.11":8}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring it back up
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 3 next-hops
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 40
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":12,"10.0.0.11":12}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bringup all the inactive nexthops at the same time
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 50
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Unused local variable

Variable memb_dict1 is not used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/611)
created_at: 2025-04-29T17:06:39Z
author_association: NONE
diff_hunk: @@ -624,6 +716,624 @@
         remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
+def fine_grained_ecmp_match_mode_prefix_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 6
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 60
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for 3 next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    # Add route with 3 next-hops
+    print("Add route with 3 next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members
+    print("Test scenarios with 3 members")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                       memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 next hop
+    print("FGNHG Bring down 1 next hop")
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # (Bring down 2 next hop and bring up 1 next hop)
+    print("Bring down 2 next hop and bring up 1 next hop")
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 60
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 next hops
+    print("Bring up 2 next hops")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 3 more next-hops
+    print("Bring up 3 more next-hops")
+    # First Resolve ARP for 3 more next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:08")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:09")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    # Test warm reboot
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 3 next-hops
+    print("Bring down 3 next-hops")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.5":20,"10.0.0.11":20}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 member and bring up 1 member at the same time
+    print("Bring down 1 member and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 2 members and bring up 1 member at the same time
+    print("Bring down 2 members and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 members and bring down 1 member at the same time
+    print("Bring up 2 members and bring down 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bringup all the inactive nexthops at the same time
+    print("Bringup all the inactive nexthops at the same time")
+    num_exp_changes = 30
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+     # Remove route
+    print("Remove route")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+
+    # validate routes and nhg member in asic db, route entry in state db are removed
+    print("validate routes and nhg member in asic db, route entry in state db are removed")
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    # Remove fgnhg prefix and group
+    print("Remove fgnhg prefix")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+    # Nothing we can wait for in terms of db entries, we sleep here
+    # to give the sw enough time to delete the entry
+    time.sleep(1)
+
+    # Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see
+    # standard(non-Fine grained) ECMP behavior
+    print("Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see")
+    print("standard(non-Fine grained) ECMP behavior")
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+            ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+    validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3)
+
+    # add back fgnhg group and prefix: The regular route should transition to fine grained ECMP
+    print("add fgnhg group and prefix: The regular route should transition to fine grained ECMP")
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    # Validate the transistion to Fine Grained ECMP
+    print("Validate the transistion to Fine Grained ECMP")
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = {}
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # remove prefix entry
+    print("remove prefix entry")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+
+    # Cleanup all FG, arp and interface
+    print("Cleanup all FG, arp and interface")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, IF_TB, ip_pref_key)
+        remove_entry(config_db, IF_TB, if_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_multi_route_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM0_NHs = 6
+    NUM1_NHs = 4
+    fg_nhg_name0 = "fgnhg0_v4"
+    fg_nhg_name1 = "fgnhg1_v4"
+    fg_nhg_prefix0 = "2.2.2.0/24"
+    fg_nhg_prefix1 = "3.3.3.0/24"
+    bucket0_size = 60
+    bucket1_size = 24
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    ### Create first fine grained next hop group and prefix
+    fvs = {"bucket_size": str(bucket0_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM0_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name0, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name0}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix0, fvs)
+
+    ### Create shared interfaces (Last two Nexthops for the 1st prefix are shared with the 2nd prefix)
+    for i in range(0,NUM0_NHs+NUM1_NHs-2):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        fvs = {"vlanid": str((i+1)*4)}
+        create_entry(config_db, VLAN_TB, vlan_name_key, fvs)
+        fvs = {"tagging_mode": "untagged"}
+        create_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key, fvs)
+        create_entry(config_db, VLAN_IF_TB, vlan_name_key, fvs_nul)
+        create_entry(config_db, VLAN_IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = vlan_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    ### Create Route for the first Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan16,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix0, fvs)
+    # No ASIC_DB entry we can wait for since ARP is not resolved yet,
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix0:
+            found_route = True
+            break
+
+    # Since we didn't populate ARP yet, route should point to RIF for kernel arp resolution to occur
+    assert (found_route == True)
+    validate_asic_nhg_router_interface(asic_db, fg_nhg_prefix0)
+
+    # Add ARP entries for both prefixes
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03")
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+    dvs.runcmd("arp -s 10.0.0.13 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.15 00:00:00:00:00:08")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 8)
+
+    # verify correct number of nhg members were created for the first prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size)
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Create 2nd next hop group and prefix
+    fvs = {"bucket_size": str(bucket1_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM1_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name1, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name1}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix1, fvs)
+
+    # Create Route for the 2nd Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.13,10.0.0.15,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan28,Vlan32,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix1, fvs)
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix1:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    # verify correct number of nhg members were created for the 2nd prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members for the first prefix and 4 members for the 2nd prefix
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict0 = {}
+    num_exp_changes = 60
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    memb_dict1 = {}
+    num_exp_changes = 24
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Test warm reboot
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    num_exp_changes = 0
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 1 common next hop
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":8,"10.0.0.15":8,"10.0.0.11":8}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring it back up
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 3 next-hops
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 40
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":12,"10.0.0.11":12}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bringup all the inactive nexthops at the same time
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 50
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # remove prefix entries
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix0)
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Variable defined multiple times

This assignment to 'asic_rt_key' is unnecessary as it is [redefined](1) before this value is used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/604)
created_at: 2025-04-29T17:06:39Z
author_association: NONE
diff_hunk: @@ -624,6 +716,624 @@
         remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
+def fine_grained_ecmp_match_mode_prefix_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 6
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 60
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for 3 next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    # Add route with 3 next-hops
+    print("Add route with 3 next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members
+    print("Test scenarios with 3 members")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                       memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 next hop
+    print("FGNHG Bring down 1 next hop")
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # (Bring down 2 next hop and bring up 1 next hop)
+    print("Bring down 2 next hop and bring up 1 next hop")
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 60
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 next hops
+    print("Bring up 2 next hops")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 3 more next-hops
+    print("Bring up 3 more next-hops")
+    # First Resolve ARP for 3 more next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:08")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:09")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    # Test warm reboot
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 3 next-hops
+    print("Bring down 3 next-hops")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.5":20,"10.0.0.11":20}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 member and bring up 1 member at the same time
+    print("Bring down 1 member and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 2 members and bring up 1 member at the same time
+    print("Bring down 2 members and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 members and bring down 1 member at the same time
+    print("Bring up 2 members and bring down 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bringup all the inactive nexthops at the same time
+    print("Bringup all the inactive nexthops at the same time")
+    num_exp_changes = 30
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+     # Remove route
+    print("Remove route")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+
+    # validate routes and nhg member in asic db, route entry in state db are removed
+    print("validate routes and nhg member in asic db, route entry in state db are removed")
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    # Remove fgnhg prefix and group
+    print("Remove fgnhg prefix")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+    # Nothing we can wait for in terms of db entries, we sleep here
+    # to give the sw enough time to delete the entry
+    time.sleep(1)
+
+    # Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see
+    # standard(non-Fine grained) ECMP behavior
+    print("Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see")
+    print("standard(non-Fine grained) ECMP behavior")
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+            ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+    validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3)
+
+    # add back fgnhg group and prefix: The regular route should transition to fine grained ECMP
+    print("add fgnhg group and prefix: The regular route should transition to fine grained ECMP")
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    # Validate the transistion to Fine Grained ECMP
+    print("Validate the transistion to Fine Grained ECMP")
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = {}
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # remove prefix entry
+    print("remove prefix entry")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+
+    # Cleanup all FG, arp and interface
+    print("Cleanup all FG, arp and interface")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, IF_TB, ip_pref_key)
+        remove_entry(config_db, IF_TB, if_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_multi_route_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM0_NHs = 6
+    NUM1_NHs = 4
+    fg_nhg_name0 = "fgnhg0_v4"
+    fg_nhg_name1 = "fgnhg1_v4"
+    fg_nhg_prefix0 = "2.2.2.0/24"
+    fg_nhg_prefix1 = "3.3.3.0/24"
+    bucket0_size = 60
+    bucket1_size = 24
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    ### Create first fine grained next hop group and prefix
+    fvs = {"bucket_size": str(bucket0_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM0_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name0, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name0}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix0, fvs)
+
+    ### Create shared interfaces (Last two Nexthops for the 1st prefix are shared with the 2nd prefix)
+    for i in range(0,NUM0_NHs+NUM1_NHs-2):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        fvs = {"vlanid": str((i+1)*4)}
+        create_entry(config_db, VLAN_TB, vlan_name_key, fvs)
+        fvs = {"tagging_mode": "untagged"}
+        create_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key, fvs)
+        create_entry(config_db, VLAN_IF_TB, vlan_name_key, fvs_nul)
+        create_entry(config_db, VLAN_IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = vlan_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    ### Create Route for the first Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan16,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix0, fvs)
+    # No ASIC_DB entry we can wait for since ARP is not resolved yet,
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix0:
+            found_route = True
+            break
+
+    # Since we didn't populate ARP yet, route should point to RIF for kernel arp resolution to occur
+    assert (found_route == True)
+    validate_asic_nhg_router_interface(asic_db, fg_nhg_prefix0)
+
+    # Add ARP entries for both prefixes
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03")
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+    dvs.runcmd("arp -s 10.0.0.13 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.15 00:00:00:00:00:08")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 8)
+
+    # verify correct number of nhg members were created for the first prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size)
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Create 2nd next hop group and prefix
+    fvs = {"bucket_size": str(bucket1_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM1_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name1, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name1}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix1, fvs)
+
+    # Create Route for the 2nd Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.13,10.0.0.15,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan28,Vlan32,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix1, fvs)
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix1:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    # verify correct number of nhg members were created for the 2nd prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members for the first prefix and 4 members for the 2nd prefix
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict0 = {}
+    num_exp_changes = 60
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    memb_dict1 = {}
+    num_exp_changes = 24
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Test warm reboot
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    num_exp_changes = 0
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 1 common next hop
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":8,"10.0.0.15":8,"10.0.0.11":8}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring it back up
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 3 next-hops
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 40
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":12,"10.0.0.11":12}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bringup all the inactive nexthops at the same time
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 50
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # remove prefix entries
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix0)
+    ps._del(fg_nhg_prefix0)
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix1)
+    ps._del(fg_nhg_prefix1)
+
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix0)
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix1)
+    # Cleanup all FG, arp and interface
+    remove_entry(config_db, "FG_NHG", fg_nhg_name0)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name1)
+
+    for i in range(0,NUM0_NHs+NUM1_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, VLAN_IF_TB, ip_pref_key)
+        remove_entry(config_db, VLAN_IF_TB, vlan_name_key)
+        remove_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key)
+        remove_entry(config_db, VLAN_TB, vlan_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_even_distribution_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 16
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 256
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Unused local variable

Variable old_log_level is not used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/612)
created_at: 2025-04-29T17:06:39Z
author_association: NONE
diff_hunk: @@ -624,6 +716,624 @@
         remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
+def fine_grained_ecmp_match_mode_prefix_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 6
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 60
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for 3 next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    # Add route with 3 next-hops
+    print("Add route with 3 next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members
+    print("Test scenarios with 3 members")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                       memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 next hop
+    print("FGNHG Bring down 1 next hop")
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # (Bring down 2 next hop and bring up 1 next hop)
+    print("Bring down 2 next hop and bring up 1 next hop")
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 60
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 next hops
+    print("Bring up 2 next hops")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 3 more next-hops
+    print("Bring up 3 more next-hops")
+    # First Resolve ARP for 3 more next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:08")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:09")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    # Test warm reboot
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 3 next-hops
+    print("Bring down 3 next-hops")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.5":20,"10.0.0.11":20}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 member and bring up 1 member at the same time
+    print("Bring down 1 member and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 2 members and bring up 1 member at the same time
+    print("Bring down 2 members and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 members and bring down 1 member at the same time
+    print("Bring up 2 members and bring down 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bringup all the inactive nexthops at the same time
+    print("Bringup all the inactive nexthops at the same time")
+    num_exp_changes = 30
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+     # Remove route
+    print("Remove route")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+
+    # validate routes and nhg member in asic db, route entry in state db are removed
+    print("validate routes and nhg member in asic db, route entry in state db are removed")
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    # Remove fgnhg prefix and group
+    print("Remove fgnhg prefix")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+    # Nothing we can wait for in terms of db entries, we sleep here
+    # to give the sw enough time to delete the entry
+    time.sleep(1)
+
+    # Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see
+    # standard(non-Fine grained) ECMP behavior
+    print("Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see")
+    print("standard(non-Fine grained) ECMP behavior")
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+            ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+    validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3)
+
+    # add back fgnhg group and prefix: The regular route should transition to fine grained ECMP
+    print("add fgnhg group and prefix: The regular route should transition to fine grained ECMP")
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    # Validate the transistion to Fine Grained ECMP
+    print("Validate the transistion to Fine Grained ECMP")
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = {}
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # remove prefix entry
+    print("remove prefix entry")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+
+    # Cleanup all FG, arp and interface
+    print("Cleanup all FG, arp and interface")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, IF_TB, ip_pref_key)
+        remove_entry(config_db, IF_TB, if_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_multi_route_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM0_NHs = 6
+    NUM1_NHs = 4
+    fg_nhg_name0 = "fgnhg0_v4"
+    fg_nhg_name1 = "fgnhg1_v4"
+    fg_nhg_prefix0 = "2.2.2.0/24"
+    fg_nhg_prefix1 = "3.3.3.0/24"
+    bucket0_size = 60
+    bucket1_size = 24
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    ### Create first fine grained next hop group and prefix
+    fvs = {"bucket_size": str(bucket0_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM0_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name0, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name0}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix0, fvs)
+
+    ### Create shared interfaces (Last two Nexthops for the 1st prefix are shared with the 2nd prefix)
+    for i in range(0,NUM0_NHs+NUM1_NHs-2):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        fvs = {"vlanid": str((i+1)*4)}
+        create_entry(config_db, VLAN_TB, vlan_name_key, fvs)
+        fvs = {"tagging_mode": "untagged"}
+        create_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key, fvs)
+        create_entry(config_db, VLAN_IF_TB, vlan_name_key, fvs_nul)
+        create_entry(config_db, VLAN_IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = vlan_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    ### Create Route for the first Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan16,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix0, fvs)
+    # No ASIC_DB entry we can wait for since ARP is not resolved yet,
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix0:
+            found_route = True
+            break
+
+    # Since we didn't populate ARP yet, route should point to RIF for kernel arp resolution to occur
+    assert (found_route == True)
+    validate_asic_nhg_router_interface(asic_db, fg_nhg_prefix0)
+
+    # Add ARP entries for both prefixes
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03")
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+    dvs.runcmd("arp -s 10.0.0.13 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.15 00:00:00:00:00:08")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 8)
+
+    # verify correct number of nhg members were created for the first prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size)
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Create 2nd next hop group and prefix
+    fvs = {"bucket_size": str(bucket1_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM1_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name1, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name1}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix1, fvs)
+
+    # Create Route for the 2nd Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.13,10.0.0.15,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan28,Vlan32,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix1, fvs)
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix1:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    # verify correct number of nhg members were created for the 2nd prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members for the first prefix and 4 members for the 2nd prefix
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict0 = {}
+    num_exp_changes = 60
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    memb_dict1 = {}
+    num_exp_changes = 24
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Test warm reboot
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    num_exp_changes = 0
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 1 common next hop
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":8,"10.0.0.15":8,"10.0.0.11":8}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring it back up
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 3 next-hops
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 40
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":12,"10.0.0.11":12}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bringup all the inactive nexthops at the same time
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 50
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # remove prefix entries
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix0)
+    ps._del(fg_nhg_prefix0)
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix1)
+    ps._del(fg_nhg_prefix1)
+
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix0)
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix1)
+    # Cleanup all FG, arp and interface
+    remove_entry(config_db, "FG_NHG", fg_nhg_name0)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name1)
+
+    for i in range(0,NUM0_NHs+NUM1_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, VLAN_IF_TB, ip_pref_key)
+        remove_entry(config_db, VLAN_IF_TB, vlan_name_key)
+        remove_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key)
+        remove_entry(config_db, VLAN_TB, vlan_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_even_distribution_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 16
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 256
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for all NUM_NHs next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    for i in range(NUM_NHs):
+        dvs.runcmd(f"arp -s 10.0.0.{1 + i * 2} 00:00:00:00:00:{1 + i * 2:02x}")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + NUM_NHs)
+
+    # Add route with NUM_NHs next-hops
+    print(f"Add route with {NUM_NHs} next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop", ",".join([f"10.0.0.{1 + i * 2}" for i in range(NUM_NHs)])),
+        ("ifname", ",".join([f"Ethernet{i * 4}" for i in range(NUM_NHs)]))])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Unused local variable

Variable nhgid is not used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/613)
created_at: 2025-04-29T17:06:39Z
author_association: NONE
diff_hunk: @@ -624,6 +716,624 @@
         remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
+def fine_grained_ecmp_match_mode_prefix_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 6
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 60
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for 3 next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    # Add route with 3 next-hops
+    print("Add route with 3 next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members
+    print("Test scenarios with 3 members")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                       memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 next hop
+    print("FGNHG Bring down 1 next hop")
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # (Bring down 2 next hop and bring up 1 next hop)
+    print("Bring down 2 next hop and bring up 1 next hop")
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 60
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 next hops
+    print("Bring up 2 next hops")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 3 more next-hops
+    print("Bring up 3 more next-hops")
+    # First Resolve ARP for 3 more next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:08")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:09")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    # Test warm reboot
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 3 next-hops
+    print("Bring down 3 next-hops")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.5":20,"10.0.0.11":20}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 member and bring up 1 member at the same time
+    print("Bring down 1 member and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 2 members and bring up 1 member at the same time
+    print("Bring down 2 members and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 members and bring down 1 member at the same time
+    print("Bring up 2 members and bring down 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bringup all the inactive nexthops at the same time
+    print("Bringup all the inactive nexthops at the same time")
+    num_exp_changes = 30
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+     # Remove route
+    print("Remove route")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+
+    # validate routes and nhg member in asic db, route entry in state db are removed
+    print("validate routes and nhg member in asic db, route entry in state db are removed")
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    # Remove fgnhg prefix and group
+    print("Remove fgnhg prefix")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+    # Nothing we can wait for in terms of db entries, we sleep here
+    # to give the sw enough time to delete the entry
+    time.sleep(1)
+
+    # Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see
+    # standard(non-Fine grained) ECMP behavior
+    print("Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see")
+    print("standard(non-Fine grained) ECMP behavior")
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+            ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+    validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3)
+
+    # add back fgnhg group and prefix: The regular route should transition to fine grained ECMP
+    print("add fgnhg group and prefix: The regular route should transition to fine grained ECMP")
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    # Validate the transistion to Fine Grained ECMP
+    print("Validate the transistion to Fine Grained ECMP")
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = {}
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # remove prefix entry
+    print("remove prefix entry")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+
+    # Cleanup all FG, arp and interface
+    print("Cleanup all FG, arp and interface")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, IF_TB, ip_pref_key)
+        remove_entry(config_db, IF_TB, if_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_multi_route_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM0_NHs = 6
+    NUM1_NHs = 4
+    fg_nhg_name0 = "fgnhg0_v4"
+    fg_nhg_name1 = "fgnhg1_v4"
+    fg_nhg_prefix0 = "2.2.2.0/24"
+    fg_nhg_prefix1 = "3.3.3.0/24"
+    bucket0_size = 60
+    bucket1_size = 24
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    ### Create first fine grained next hop group and prefix
+    fvs = {"bucket_size": str(bucket0_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM0_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name0, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name0}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix0, fvs)
+
+    ### Create shared interfaces (Last two Nexthops for the 1st prefix are shared with the 2nd prefix)
+    for i in range(0,NUM0_NHs+NUM1_NHs-2):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        fvs = {"vlanid": str((i+1)*4)}
+        create_entry(config_db, VLAN_TB, vlan_name_key, fvs)
+        fvs = {"tagging_mode": "untagged"}
+        create_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key, fvs)
+        create_entry(config_db, VLAN_IF_TB, vlan_name_key, fvs_nul)
+        create_entry(config_db, VLAN_IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = vlan_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    ### Create Route for the first Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan16,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix0, fvs)
+    # No ASIC_DB entry we can wait for since ARP is not resolved yet,
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix0:
+            found_route = True
+            break
+
+    # Since we didn't populate ARP yet, route should point to RIF for kernel arp resolution to occur
+    assert (found_route == True)
+    validate_asic_nhg_router_interface(asic_db, fg_nhg_prefix0)
+
+    # Add ARP entries for both prefixes
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03")
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+    dvs.runcmd("arp -s 10.0.0.13 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.15 00:00:00:00:00:08")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 8)
+
+    # verify correct number of nhg members were created for the first prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size)
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Create 2nd next hop group and prefix
+    fvs = {"bucket_size": str(bucket1_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM1_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name1, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name1}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix1, fvs)
+
+    # Create Route for the 2nd Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.13,10.0.0.15,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan28,Vlan32,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix1, fvs)
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix1:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    # verify correct number of nhg members were created for the 2nd prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members for the first prefix and 4 members for the 2nd prefix
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict0 = {}
+    num_exp_changes = 60
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    memb_dict1 = {}
+    num_exp_changes = 24
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Test warm reboot
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    num_exp_changes = 0
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 1 common next hop
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":8,"10.0.0.15":8,"10.0.0.11":8}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring it back up
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 3 next-hops
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 40
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":12,"10.0.0.11":12}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bringup all the inactive nexthops at the same time
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 50
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # remove prefix entries
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix0)
+    ps._del(fg_nhg_prefix0)
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix1)
+    ps._del(fg_nhg_prefix1)
+
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix0)
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix1)
+    # Cleanup all FG, arp and interface
+    remove_entry(config_db, "FG_NHG", fg_nhg_name0)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name1)
+
+    for i in range(0,NUM0_NHs+NUM1_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, VLAN_IF_TB, ip_pref_key)
+        remove_entry(config_db, VLAN_IF_TB, vlan_name_key)
+        remove_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key)
+        remove_entry(config_db, VLAN_TB, vlan_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_even_distribution_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 16
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 256
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for all NUM_NHs next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    for i in range(NUM_NHs):
+        dvs.runcmd(f"arp -s 10.0.0.{1 + i * 2} 00:00:00:00:00:{1 + i * 2:02x}")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + NUM_NHs)
+
+    # Add route with NUM_NHs next-hops
+    print(f"Add route with {NUM_NHs} next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop", ",".join([f"10.0.0.{1 + i * 2}" for i in range(NUM_NHs)])),
+        ("ifname", ",".join([f"Ethernet{i * 4}" for i in range(NUM_NHs)]))])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    ### Start with NUM_NHs members and verify that distribution is even after a series of add/remove operations
+    for _ in range(50):
+        print()
+        print(f"Iteration: {_}, started test with {NUM_NHs} members, bucket_size: {bucket_size}")
+        nh_ips = [f"10.0.0.{1 + i * 2}" for i in range(NUM_NHs)]
+        program_route_and_validate_distribtution(app_db.db_connection, state_db, ip_to_if_map,
+                                                         fg_nhg_prefix, nh_ips, bucket_size)
+
+        ### remove 1 to 3 nexthops in each step and verify that distribution is even
+        removed_ips = []
+        while len(nh_ips) > 1:
+            num_to_remove = random.randint(1, min(3, len(nh_ips) - 1))
+            removed_in_iteration = []
+            for _ in range(num_to_remove):
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Nested loops with same variable

Nested for statement uses loop variable '_' of enclosing [for statement](1).

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/617)
created_at: 2025-04-29T17:06:39Z
author_association: NONE
diff_hunk: @@ -624,6 +716,624 @@
         remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
+def fine_grained_ecmp_match_mode_prefix_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 6
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 60
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for 3 next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    # Add route with 3 next-hops
+    print("Add route with 3 next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members
+    print("Test scenarios with 3 members")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                       memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 next hop
+    print("FGNHG Bring down 1 next hop")
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # (Bring down 2 next hop and bring up 1 next hop)
+    print("Bring down 2 next hop and bring up 1 next hop")
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 60
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 next hops
+    print("Bring up 2 next hops")
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 3 more next-hops
+    print("Bring up 3 more next-hops")
+    # First Resolve ARP for 3 more next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:08")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:09")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 3)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+    # Test warm reboot
+    print("Test warm reboot")
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+    nh_oid_map = get_nh_oid_map(asic_db)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 0
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 3 next-hops
+    print("Bring down 3 next-hops")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.5":20,"10.0.0.11":20}
+    num_exp_changes = 30
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 1 member and bring up 1 member at the same time
+    print("Bring down 1 member and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring down 2 members and bring up 1 member at the same time
+    print("Bring down 2 members and bring up 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bring up 2 members and bring down 1 member at the same time
+    print("Bring up 2 members and bring down 1 member at the same time")
+    nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.11":20}
+    num_exp_changes = 40
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # Bringup all the inactive nexthops at the same time
+    print("Bringup all the inactive nexthops at the same time")
+    num_exp_changes = 30
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+     # Remove route
+    print("Remove route")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+
+    # validate routes and nhg member in asic db, route entry in state db are removed
+    print("validate routes and nhg member in asic db, route entry in state db are removed")
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    # Remove fgnhg prefix and group
+    print("Remove fgnhg prefix")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+    # Nothing we can wait for in terms of db entries, we sleep here
+    # to give the sw enough time to delete the entry
+    time.sleep(1)
+
+    # Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see
+    # standard(non-Fine grained) ECMP behavior
+    print("Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see")
+    print("standard(non-Fine grained) ECMP behavior")
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+            ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
+    ps.set(fg_nhg_prefix, fvs)
+    validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3)
+
+    # add back fgnhg group and prefix: The regular route should transition to fine grained ECMP
+    print("add fgnhg group and prefix: The regular route should transition to fine grained ECMP")
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    # Validate the transistion to Fine Grained ECMP
+    print("Validate the transistion to Fine Grained ECMP")
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    nh_oid_map = {}
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict = {}
+    num_exp_changes = 60
+    memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+
+    # remove prefix entry
+    print("remove prefix entry")
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
+    ps._del(fg_nhg_prefix)
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+
+    # Cleanup all FG, arp and interface
+    print("Cleanup all FG, arp and interface")
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, IF_TB, ip_pref_key)
+        remove_entry(config_db, IF_TB, if_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_multi_route_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM0_NHs = 6
+    NUM1_NHs = 4
+    fg_nhg_name0 = "fgnhg0_v4"
+    fg_nhg_name1 = "fgnhg1_v4"
+    fg_nhg_prefix0 = "2.2.2.0/24"
+    fg_nhg_prefix1 = "3.3.3.0/24"
+    bucket0_size = 60
+    bucket1_size = 24
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    ### Create first fine grained next hop group and prefix
+    fvs = {"bucket_size": str(bucket0_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM0_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name0, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name0}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix0, fvs)
+
+    ### Create shared interfaces (Last two Nexthops for the 1st prefix are shared with the 2nd prefix)
+    for i in range(0,NUM0_NHs+NUM1_NHs-2):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        fvs = {"vlanid": str((i+1)*4)}
+        create_entry(config_db, VLAN_TB, vlan_name_key, fvs)
+        fvs = {"tagging_mode": "untagged"}
+        create_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key, fvs)
+        create_entry(config_db, VLAN_IF_TB, vlan_name_key, fvs_nul)
+        create_entry(config_db, VLAN_IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = vlan_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    ### Create Route for the first Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan16,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix0, fvs)
+    # No ASIC_DB entry we can wait for since ARP is not resolved yet,
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix0:
+            found_route = True
+            break
+
+    # Since we didn't populate ARP yet, route should point to RIF for kernel arp resolution to occur
+    assert (found_route == True)
+    validate_asic_nhg_router_interface(asic_db, fg_nhg_prefix0)
+
+    # Add ARP entries for both prefixes
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01")
+    dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02")
+    dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03")
+    dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
+    dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05")
+    dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06")
+    dvs.runcmd("arp -s 10.0.0.13 00:00:00:00:00:07")
+    dvs.runcmd("arp -s 10.0.0.15 00:00:00:00:00:08")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 8)
+
+    # verify correct number of nhg members were created for the first prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size)
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Create 2nd next hop group and prefix
+    fvs = {"bucket_size": str(bucket1_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM1_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name1, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name1}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix1, fvs)
+
+    # Create Route for the 2nd Prefix
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.13,10.0.0.15,10.0.0.9,10.0.0.11"),
+        ("ifname", "Vlan28,Vlan32,Vlan20,Vlan24")])
+    ps.set(fg_nhg_prefix1, fvs)
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix1:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    # verify correct number of nhg members were created for the 2nd prefix
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    ### Test scenarios with 3 members for the first prefix and 4 members for the 2nd prefix
+
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    memb_dict0 = {}
+    num_exp_changes = 60
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    memb_dict1 = {}
+    num_exp_changes = 24
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Test warm reboot
+    run_warm_reboot(dvs)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket0_size+bucket1_size)
+
+    nh_oid_map = get_nh_oid_map(asic_db)
+
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 0
+    memb_dict0 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    num_exp_changes = 0
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    memb_dict1 = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                    memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 1 common next hop
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":8,"10.0.0.15":8,"10.0.0.11":8}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring it back up
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
+    num_exp_changes = 20
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 6
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bring down 3 next-hops
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.9":60}
+    num_exp_changes = 40
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":12,"10.0.0.11":12}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # Bringup all the inactive nexthops at the same time
+    nhgid0 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix0, bucket0_size)
+    nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10}
+    num_exp_changes = 50
+    memb_dict0 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict0, num_exp_changes, fg_nhg_prefix0, nh_memb_exp_count, nh_oid_map, nhgid0, bucket0_size)
+
+    nhgid1 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix1, bucket1_size)
+    nh_memb_exp_count = {"10.0.0.13":6,"10.0.0.15":6,"10.0.0.9":6,"10.0.0.11":6}
+    num_exp_changes = 12
+    memb_dict1 = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                        memb_dict1, num_exp_changes, fg_nhg_prefix1, nh_memb_exp_count, nh_oid_map, nhgid1, bucket1_size)
+
+    # remove prefix entries
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix0)
+    ps._del(fg_nhg_prefix0)
+    asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix1)
+    ps._del(fg_nhg_prefix1)
+
+    asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+    state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
+
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix0)
+    remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix1)
+    # Cleanup all FG, arp and interface
+    remove_entry(config_db, "FG_NHG", fg_nhg_name0)
+    remove_entry(config_db, "FG_NHG", fg_nhg_name1)
+
+    for i in range(0,NUM0_NHs+NUM1_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        vlan_name_key = "Vlan" + str((i+1)*4)
+        ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31"
+        remove_entry(config_db, VLAN_IF_TB, ip_pref_key)
+        remove_entry(config_db, VLAN_IF_TB, vlan_name_key)
+        remove_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key)
+        remove_entry(config_db, VLAN_TB, vlan_name_key)
+        dvs.port_admin_set(if_name_key, "down")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+
+def fine_grained_ecmp_match_mode_prefix_even_distribution_test(dvs):
+    app_db = dvs.get_app_db()
+    asic_db = dvs.get_asic_db()
+    config_db = dvs.get_config_db()
+    state_db = dvs.get_state_db()
+    fvs_nul = {"NULL": "NULL"}
+    NUM_NHs = 16
+    fg_nhg_name = "fgnhg_v4"
+    fg_nhg_prefix = "2.2.2.0/24"
+    bucket_size = 256
+    ip_to_if_map = {}
+    match_mode = 'prefix-based'
+
+    # Update log level so that we can analyze the log in case the test failed
+    logfvs = config_db.wait_for_entry("LOGGER", "orchagent")
+    old_log_level = logfvs.get("LOGLEVEL")
+    logfvs["LOGLEVEL"] = "INFO"
+    config_db.update_entry("LOGGER", "orchagent", logfvs)
+
+    fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode,
+           "max_next_hops": str(NUM_NHs)}
+    create_entry(config_db, FG_NHG, fg_nhg_name, fvs)
+
+    fvs = {"FG_NHG": fg_nhg_name}
+    create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
+
+    for i in range(0,NUM_NHs):
+        if_name_key = "Ethernet" + str(i*4)
+        ip_pref_key = if_name_key + "|10.0.0." + str(i*2) + "/31"
+        create_entry(config_db, IF_TB, if_name_key, fvs_nul)
+        create_entry(config_db, IF_TB, ip_pref_key, fvs_nul)
+        dvs.port_admin_set(if_name_key, "up")
+        dvs.servers[i].runcmd("ip link set down dev eth0") == 0
+        dvs.servers[i].runcmd("ip link set up dev eth0") == 0
+        ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
+
+    # Wait for the software to receive the entries
+    time.sleep(1)
+
+    # Resolve ARP for all NUM_NHs next-hops
+    asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB))
+    for i in range(NUM_NHs):
+        dvs.runcmd(f"arp -s 10.0.0.{1 + i * 2} 00:00:00:00:00:{1 + i * 2:02x}")
+
+    asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + NUM_NHs)
+
+    # Add route with NUM_NHs next-hops
+    print(f"Add route with {NUM_NHs} next-hops")
+    ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
+    fvs = swsscommon.FieldValuePairs([("nexthop", ",".join([f"10.0.0.{1 + i * 2}" for i in range(NUM_NHs)])),
+        ("ifname", ",".join([f"Ethernet{i * 4}" for i in range(NUM_NHs)]))])
+    ps.set(fg_nhg_prefix, fvs)
+
+    # We just use sleep so that the sw receives this entry
+    time.sleep(1)
+
+    adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+    rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB)
+    keys = rtbl.getKeys()
+    found_route = False
+    for k in keys:
+        rt_key = json.loads(k)
+
+        if rt_key['dest'] == fg_nhg_prefix:
+            found_route = True
+            break
+
+    assert (found_route == True)
+
+    asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
+    nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size)
+
+    ### Start with NUM_NHs members and verify that distribution is even after a series of add/remove operations
+    for _ in range(50):
+        print()
+        print(f"Iteration: {_}, started test with {NUM_NHs} members, bucket_size: {bucket_size}")
+        nh_ips = [f"10.0.0.{1 + i * 2}" for i in range(NUM_NHs)]
+        program_route_and_validate_distribtution(app_db.db_connection, state_db, ip_to_if_map,
+                                                         fg_nhg_prefix, nh_ips, bucket_size)
+
+        ### remove 1 to 3 nexthops in each step and verify that distribution is even
+        removed_ips = []
+        while len(nh_ips) > 1:
+            num_to_remove = random.randint(1, min(3, len(nh_ips) - 1))
+            removed_in_iteration = []
+            for _ in range(num_to_remove):
+                removed_ip = nh_ips.pop()
+                removed_ips.append(removed_ip)
+                removed_in_iteration.append(removed_ip)
+            print(f"Removed IPs: {removed_in_iteration}, No. of remaining IPs: {len(nh_ips)}")
+            program_route_and_validate_distribtution(app_db.db_connection, state_db, ip_to_if_map,
+                                                         fg_nhg_prefix, nh_ips, bucket_size)
+
+        ### add 1-3 nexthops in each step and verify that distribution stays even
+        while len(nh_ips) < NUM_NHs:
+            num_to_add = random.randint(1, min(3, len(removed_ips)))
+            added_in_iteration = []
+            for _ in range(num_to_add):
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Nested loops with same variable

Nested for statement uses loop variable '_' of enclosing [for statement](1).

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/618)
created_at: 2025-04-29T17:06:40Z
author_association: NONE
diff_hunk: @@ -680,49 +1408,58 @@
         nh_oid_map = get_nh_oid_map(asic_db)
 
         # The route had been created with 0 members in bank
+        memb_dict = {}
         nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.11":64}
-        validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                                fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128
+        memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
+                        memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Add 2 nhs to both bank 0 and bank 1
         nh_memb_exp_count = {"10.0.0.1":22,"10.0.0.3":21,"10.0.0.5":21,"10.0.0.11":22,
                 "10.0.0.13":21,"10.0.0.15":21}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-22-22
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Add 2 more nhs to both bank 0 and bank 1
         nh_memb_exp_count = {"10.0.0.1":13,"10.0.0.3":13,"10.0.0.5":13,"10.0.0.7":12,
                 "10.0.0.9":13,"10.0.0.11":13,"10.0.0.13":13,"10.0.0.15":13,"10.0.0.17":12,"10.0.0.19":13}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-6*13
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Remove 1 nh from bank 0 and remove 2 nhs from bank 1
         nh_memb_exp_count = {"10.0.0.3":16,"10.0.0.5":16,"10.0.0.7":16,"10.0.0.9":16,
                 "10.0.0.11":22,"10.0.0.13":21,"10.0.0.19":21}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-13-13-12-13-13-13-13
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Remove 1 nh from bank 0 and add 1 nh to bank 1
         nh_memb_exp_count = {"10.0.0.3":22,"10.0.0.7":21,"10.0.0.9":21,"10.0.0.13":16,
                 "10.0.0.15":16,"10.0.0.17":16,"10.0.0.19":16}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-5*16
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Remove 2 nh from bank 0 and remove 3 nh from bank 1
         nh_memb_exp_count = {"10.0.0.7":64,"10.0.0.11":64}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-21
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Add 2 nhs to bank 0 and remove all nh from bank 1
         nh_memb_exp_count = {"10.0.0.5":42,"10.0.0.7":44,"10.0.0.9":42}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-22
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
+                    memb_dict, num_exp_changes, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
 
         # Add 2 nhs to bank 0 and add 1 nh to bank 1
         nh_memb_exp_count = {"10.0.0.1":12,"10.0.0.3":13,"10.0.0.5":13,"10.0.0.7":13,
                 "10.0.0.9":13,"10.0.0.11":64}
-        program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
-                            fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
+        num_exp_changes = 128-13-13-13
+        memb_dict = program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map,
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Unused local variable

Variable memb_dict is not used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/614)
created_at: 2025-04-29T17:06:40Z
author_association: NONE
diff_hunk: @@ -850,8 +1593,9 @@
         asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size)
         # Ensure that 2nd route is still here and then delete it
         nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.5":64}
-        validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
-                                fg_nhg_prefix_2, nh_memb_exp_count, nh_oid_map, nhgid_2, bucket_size)
+        num_exp_changes = 0
+        memb_dict = validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
path: tests/test_fgnhg.py
user: login: github-advanced-security[bot]
body: ## Unused local variable

Variable memb_dict is not used.

[Show more details](https://github.com/sonic-net/sonic-swss/security/code-scanning/615)
created_at: 2025-04-29T17:06:40Z
author_association: NONE

PR comments:
user: login: mssonicbld
created_at: 2025-01-29T01:28:50Z
body: /azp run
author_association: COLLABORATOR
user: login: linux-foundation-easycla[bot]
created_at: 2025-01-29T01:28:51Z
body: <a href="https://easycla.lfx.linuxfoundation.org/#/?version=2"><img src="https://s3.amazonaws.com/cla-project-logo-prod/cla-signed.svg" alt="CLA Signed" align="left" height="28" width="328" ></a><br/><br />The committers listed above are authorized under a signed CLA.<ul><li>:white_check_mark: login: ashutosh-agrawal / name: Ashutosh Agrawal  (ea659ba5b6d9260b64557342cbedf5e5e5cc857e, 95a1179a52d4fcd634405f586b7fda7ef727355f, 883b012e296069c3eda606a20627a67f9c38eede, b62fbb685b20fe95805c465b9ee03ccee7e61e81, fa7edbe3d332a9a689fb7e609393396c2bd19e7a, 598e9f28fc43d880529151dd01e0dbac963e53e6, a0c27433abee703c82d58cc734f84442925b308d, a24f334731cf841c3b4ae04cdac53a50c68eabc7, cebcef59cf0610f6479b8c61ebbed4ccd3e59616)</li><li>:white_check_mark: login: manamand2020  (b5e9ffb3add3e037fcab67119d84326f02bbbde3)</li></ul><!-- Date Modified: 2025-04-28 17:09:01.008615 -->
author_association: NONE
user: login: azure-pipelines[bot]
created_at: 2025-01-29T01:29:01Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-01-29T01:47:25Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-01-29T01:47:34Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-01-29T07:13:52Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-01-29T07:14:02Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-02-04T08:24:47Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-02-04T08:24:57Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-02-04T18:16:49Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-02-04T18:17:01Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: manamand2020
created_at: 2025-02-04T23:08:32Z
body: /azp run
author_association: CONTRIBUTOR
user: login: azure-pipelines[bot]
created_at: 2025-02-04T23:08:38Z
body: <samp>
Commenter does not have sufficient privileges for PR 3488 in repo sonic-net/sonic-swss<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-02-19T06:08:07Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-02-19T06:08:18Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-03-15T00:38:04Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-03-15T00:38:13Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: ashutosh-agrawal
created_at: 2025-04-02T16:22:37Z
body: @anish-n We have responded to most of the PR comments.  If you find any of the responses satisfactory, please mark those comments as resolved. So, we can focus on resolving the remaining comments. 
author_association: MEMBER
user: login: anish-n
created_at: 2025-04-03T00:31:24Z
body: > @anish-n We have responded to most of the PR comments. If you find any of the responses satisfactory, please mark those comments as resolved. So, we can focus on resolving the remaining comments.

Sounds good, will do, I will do the next round of review by 4/7
author_association: CONTRIBUTOR
user: login: mssonicbld
created_at: 2025-04-11T18:08:05Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-11T18:08:15Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-04-11T18:20:10Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-11T18:20:20Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-04-11T21:52:07Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-11T21:52:17Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-04-14T20:44:29Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-14T20:44:35Z
body: <samp>
Azure Pipelines will not run the associated pipelines, because the pull request was updated after the run command was issued. Review the pull request again and issue a new run command.<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-04-14T23:35:29Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-14T23:35:35Z
body: <samp>
Azure Pipelines will not run the associated pipelines, because the pull request was updated after the run command was issued. Review the pull request again and issue a new run command.<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-04-14T23:46:42Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-14T23:46:48Z
body: <samp>
Azure Pipelines will not run the associated pipelines, because the pull request was updated after the run command was issued. Review the pull request again and issue a new run command.<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-04-15T17:38:00Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-15T17:38:06Z
body: <samp>
Azure Pipelines will not run the associated pipelines, because the pull request was updated after the run command was issued. Review the pull request again and issue a new run command.<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-04-15T17:46:15Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-15T17:46:25Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-04-19T00:55:12Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-19T00:55:21Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-04-24T19:00:25Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-24T19:00:38Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: mssonicbld
created_at: 2025-04-28T17:09:04Z
body: /azp run
author_association: COLLABORATOR
user: login: azure-pipelines[bot]
created_at: 2025-04-28T17:09:15Z
body: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
author_association: NONE
user: login: prsunny
created_at: 2025-04-29T17:04:27Z
body: approved to merge
author_association: COLLABORATOR