Task: Bunch of changes to support MLMG performance analysis and tuning (AMReX-Codes/amrex#388)

org: AMReX-Codes
repo: amrex
pr_number: 388
issue_title: Bunch of changes to support MLMG performance analysis and tuning
resolved_issues: number: 393
title: Weiqun/perf
body: some minor changes


Reference solution:
fix_patch: diff --git a/Src/Base/AMReX.cpp b/Src/Base/AMReX.cpp
index d42c0cca2d4..ec0a2f92e94 100644
--- a/Src/Base/AMReX.cpp
+++ b/Src/Base/AMReX.cpp
@@ -20,6 +20,8 @@
 
 #include <AMReX_Gpu.H>
 
+#include <AMReX_Machine.H>
+
 #ifdef AMREX_USE_EB
 #include <AMReX_EB2.H>
 #endif
@@ -533,6 +535,7 @@ amrex::Initialize (int& argc, char**& argv, bool build_parm_parse,
 #endif
     BL_PROFILE_INITPARAMS();
 #endif
+    machine::Initialize();
 
     if (double(std::numeric_limits<long>::max()) < 9.e18)
     {
diff --git a/Src/Base/AMReX_DistributionMapping.H b/Src/Base/AMReX_DistributionMapping.H
index 7e399dab103..b7c7f38ed15 100644
--- a/Src/Base/AMReX_DistributionMapping.H
+++ b/Src/Base/AMReX_DistributionMapping.H
@@ -156,7 +156,9 @@ class DistributionMapping
     static DistributionMapping makeRoundRobin (const MultiFab& weight);
     static DistributionMapping makeSFC        (const MultiFab& weight, bool sort=true);
 
-    static std::vector<std::vector<int> > makeSFC (const BoxArray& ba);
+    // if use_box_vol is true, weight boxes by their volume in Distribute
+    // otherwise, all boxes will be treated with equal weight
+    static std::vector<std::vector<int> > makeSFC (const BoxArray& ba, bool use_box_vol=true);
 
 private:
 
diff --git a/Src/Base/AMReX_DistributionMapping.cpp b/Src/Base/AMReX_DistributionMapping.cpp
index 009f265a004..29a2fe32e55 100644
--- a/Src/Base/AMReX_DistributionMapping.cpp
+++ b/Src/Base/AMReX_DistributionMapping.cpp
@@ -24,6 +24,10 @@
 #include <cstring>
 #include <iomanip>
 
+namespace {
+int flag_verbose_mapper;
+}
+
 namespace amrex {
 
     bool initialized = false;
@@ -111,14 +115,16 @@ DistributionMapping::Initialize ()
     sfc_threshold    = 0;
     max_efficiency   = 0.9;
     node_size        = 0;
+    flag_verbose_mapper = 0;
 
     ParmParse pp("DistributionMapping");
 
-    pp.query("v"      ,          verbose);
-    pp.query("verbose",          verbose);
-    pp.query("efficiency",       max_efficiency);
-    pp.query("sfc_threshold",    sfc_threshold);
-    pp.query("node_size",        node_size);
+    pp.query("v"      ,             verbose);
+    pp.query("verbose",             verbose);
+    pp.query("efficiency",          max_efficiency);
+    pp.query("sfc_threshold",       sfc_threshold);
+    pp.query("node_size",           node_size);
+    pp.query("verbose_mapper",      flag_verbose_mapper);
 
     std::string theStrategy;
 
@@ -192,21 +198,10 @@ DistributionMapping::LeastUsedCPUs (int         nprocs,
     BL_PROFILE("DistributionMapping::LeastUsedCPUs()");
 
     AMREX_ASSERT(nprocs <= ParallelContext::NProcsSub());
-    Vector<long> bytes(ParallelContext::NProcsSub());
 
+    Vector<long> bytes(ParallelContext::NProcsSub());
     long thisbyte = amrex::TotalBytesAllocatedInFabs()/1024;
-
-    BL_COMM_PROFILE(BLProfiler::Allgather, sizeof(long), BLProfiler::BeforeCall(),
-                    BLProfiler::NoTag());
-    MPI_Allgather(&thisbyte,
-                  1,
-                  ParallelDescriptor::Mpi_typemap<long>::type(),
-                  bytes.dataPtr(),
-                  1,
-                  ParallelDescriptor::Mpi_typemap<long>::type(),
-                  ParallelContext::CommunicatorSub());
-    BL_COMM_PROFILE(BLProfiler::Allgather, sizeof(long), BLProfiler::AfterCall(),
-                    BLProfiler::NoTag());
+    ParallelAllGather::AllGather(thisbyte, bytes.dataPtr(), ParallelContext::CommunicatorSub());
 
     std::vector<LIpair> LIpairV;
 
@@ -225,6 +220,13 @@ DistributionMapping::LeastUsedCPUs (int         nprocs,
     {
         result[i] = LIpairV[i].second;
     }
+
+    if (flag_verbose_mapper) {
+        Print() << "LeastUsedCPUs:" << std::endl;
+        for (const auto &p : LIpairV) {
+            Print() << "  Rank " << p.second << " contains " << p.first << std::endl;
+        }
+    }
 #else
     for (int i(0); i < nprocs; ++i)
     {
@@ -244,21 +246,9 @@ DistributionMapping::LeastUsedTeams (Vector<int>        & rteam,
 
     AMREX_ALWAYS_ASSERT(ParallelContext::CommunicatorSub() == ParallelDescriptor::Communicator());
 
-    Vector<long> bytes(ParallelDescriptor::NProcs());
-
+    Vector<long> bytes(ParallelContext::NProcsSub());
     long thisbyte = amrex::TotalBytesAllocatedInFabs()/1024;
-
-    BL_COMM_PROFILE(BLProfiler::Allgather, sizeof(long), BLProfiler::BeforeCall(),
-                    BLProfiler::NoTag());
-    MPI_Allgather(&thisbyte,
-                  1,
-                  ParallelDescriptor::Mpi_typemap<long>::type(),
-                  bytes.dataPtr(),
-                  1,
-                  ParallelDescriptor::Mpi_typemap<long>::type(),
-                  ParallelDescriptor::Communicator());
-    BL_COMM_PROFILE(BLProfiler::Allgather, sizeof(long), BLProfiler::AfterCall(),
-                    BLProfiler::NoTag());
+    ParallelAllGather::AllGather(thisbyte, bytes.dataPtr(), ParallelContext::CommunicatorSub());
 
     std::vector<LIpair> LIpairV;
     std::vector<LIpair> LIworker;
@@ -397,6 +387,10 @@ DistributionMapping::RoundRobinDoIt (int                  nboxes,
                                      int                 /* nprocs */,
                                      std::vector<LIpair>* LIpairV)
 {
+    if (flag_verbose_mapper) {
+        Print() << "DM: RoundRobinDoIt called..." << std::endl;
+    }
+
     int nprocs = ParallelContext::NProcsSub();
 
     // If team is not use, we are going to treat it as a special case in which
@@ -435,6 +429,10 @@ DistributionMapping::RoundRobinDoIt (int                  nboxes,
 	    int wid = (w[tid]++) % nworkers;
 	    int rank = tid*nworkers + wrkerord[tid][wid];
 	    m_ref->m_pmap[(*LIpairV)[i].second] = ParallelContext::local_to_global_rank(rank);
+            if (flag_verbose_mapper) {
+                Print() << "  Mapping box " << (*LIpairV)[i].second << " of size "
+                        << (*LIpairV)[i].first << " to rank " << rank << std::endl;
+            }
 	}
     }
     else
@@ -445,6 +443,9 @@ DistributionMapping::RoundRobinDoIt (int                  nboxes,
 	    int wid = (w[tid]++) % nworkers;
 	    int rank = tid*nworkers + wrkerord[tid][wid];
 	    m_ref->m_pmap[i] = ParallelContext::local_to_global_rank(rank);
+            if (flag_verbose_mapper) {
+                Print() << "  Mapping box " << i << " to rank " << rank << std::endl;
+            }
 	}
     }
 }
@@ -721,6 +722,10 @@ DistributionMapping::KnapSackDoIt (const std::vector<long>& wgts,
                                    bool                     do_full_knapsack,
 				   int                      nmax)
 {
+    if (flag_verbose_mapper) {
+        Print() << "DM: KnapSackDoIt called..." << std::endl;
+    }
+
     BL_PROFILE("DistributionMapping::KnapSackDoIt()");
 
     int nprocs = ParallelContext::NProcsSub();
@@ -741,6 +746,15 @@ DistributionMapping::KnapSackDoIt (const std::vector<long>& wgts,
 
     knapsack(wgts,nteams,vec,efficiency,do_full_knapsack,nmax);
 
+    if (flag_verbose_mapper) {
+        for (int i = 0; i < vec.size(); ++i) {
+            Print() << "  Bucket " << i << " contains boxes:" << std::endl;
+            for (int j = 0; j < vec[i].size(); ++j) {
+                Print() << "    " << vec[i][j] << std::endl;
+            }
+        }
+    }
+
     BL_ASSERT(static_cast<int>(vec.size()) == nteams);
 
     std::vector<LIpair> LIpairV;
@@ -761,6 +775,12 @@ DistributionMapping::KnapSackDoIt (const std::vector<long>& wgts,
 
     Sort(LIpairV, true);
 
+    if (flag_verbose_mapper) {
+        for (const auto &p : LIpairV) {
+            Print() << "  Bucket " << p.second << " total weight: " << p.first << std::endl;
+        }
+    }
+
     Vector<int> ord;
     Vector<Vector<int> > wrkerord;
 
@@ -783,6 +803,10 @@ DistributionMapping::KnapSackDoIt (const std::vector<long>& wgts,
 	const std::vector<int>& vi = vec[idx];
 	const int N = vi.size();
 
+        if (flag_verbose_mapper) {
+            Print() << "  Mapping bucket " << idx << " to rank " << tid << std::endl;
+        }
+
 	if (nteams == nprocs) {
 	    for (int j = 0; j < N; ++j)
 	    {
@@ -917,6 +941,21 @@ Distribute (const std::vector<SFCToken>&     tokens,
             std::vector< std::vector<int> >& v)
 
 {
+    BL_PROFILE("DistributionMapping::Distribute()");
+
+    if (flag_verbose_mapper) {
+        Print() << "Distribute:" << std::endl;
+        Print() << "  volpercpu: " << volpercpu << std::endl;
+        Print() << "  Sorted SFC Tokens:" << std::endl;
+        int idx = 0;
+        for (const auto &t : tokens) {
+            Print() << "    " << idx++ << ": "
+                    << t.m_box << ": "
+                    << t.m_idx << ": "
+                    << t.m_vol << std::endl;
+        }
+    }
+
     BL_ASSERT(static_cast<int>(v.size()) == nprocs);
 
     int  K        = 0;
@@ -949,6 +988,26 @@ Distribute (const std::vector<SFCToken>&     tokens,
         }
     }
 
+    if (flag_verbose_mapper) {
+        Print() << "Distributed SFC Tokens:" << std::endl;
+        int idx = 0;
+        for (int i = 0; i < nprocs; ++i) {
+            Print() << "  Rank/Team " << i << ":" << std::endl;
+            Real rank_vol = 0;
+            for (const auto &box : v[i]) {
+                const auto &t = tokens[idx];
+                BL_ASSERT(box == t.m_box);
+                Print() << "    " << idx << ": "
+                        << t.m_box << ": "
+                        << t.m_idx << ": "
+                        << t.m_vol << std::endl;
+                rank_vol += t.m_vol;
+                idx++;
+            }
+            Print() << "    Total Rank Vol: " << rank_vol << std::endl;
+        }
+    }
+
 #ifdef AMREX_DEBUG
     int cnt = 0;
     for (int i = 0; i < nprocs; ++i) {
@@ -964,6 +1023,10 @@ DistributionMapping::SFCProcessorMapDoIt (const BoxArray&          boxes,
                                           int                   /*   nprocs */,
                                           bool                     sort)
 {
+    if (flag_verbose_mapper) {
+        Print() << "DM: SFCProcessorMapDoIt called..." << std::endl;
+    }
+
     BL_PROFILE("DistributionMapping::SFCProcessorMapDoIt()");
 
     int nprocs = ParallelContext::NProcsSub();
@@ -984,6 +1047,11 @@ DistributionMapping::SFCProcessorMapDoIt (const BoxArray&          boxes,
     }
 #endif
 
+    if (flag_verbose_mapper) {
+        Print() << "  (nprocs, nteams, nworkers) = ("
+                << nprocs << ", " << nteams << ", " << nworkers << ")\n";
+    }
+
     std::vector<SFCToken> tokens;
 
     const int N = boxes.size();
@@ -1048,6 +1116,12 @@ DistributionMapping::SFCProcessorMapDoIt (const BoxArray&          boxes,
 
     if (sort) Sort(LIpairV, true);
 
+    if (flag_verbose_mapper) {
+        for (const auto &p : LIpairV) {
+            Print() << "  Bucket " << p.second << " contains " << p.first << std::endl;
+        }
+    }
+
     // LIpairV has a size of nteams and LIpairV[] is pair whose first is weight
     // and second is an index into vec.  LIpairV is sorted by weight such that
     // LIpairV is the heaviest.
@@ -1086,6 +1160,10 @@ DistributionMapping::SFCProcessorMapDoIt (const BoxArray&          boxes,
         const std::vector<int>& vi = vec[ivec];   // this vector contains boxes assigned to this team
 	const int Nbx = vi.size();                // # of boxes assigned to this team
 
+        if (flag_verbose_mapper) {
+            Print() << "Mapping bucket " << LIpairV[i].second << " to rank " << ord[i] << std::endl;
+        }
+
 	if (nteams == nprocs) { // In this case, team id is process id.
 	    for (int j = 0; j < Nbx; ++j)
 	    {
@@ -1136,7 +1214,7 @@ DistributionMapping::SFCProcessorMapDoIt (const BoxArray&          boxes,
 	}
     }
 
-    if (verbose && ParallelDescriptor::IOProcessor())
+    if (verbose)
     {
         Real sum_wgt = 0, max_wgt = 0;
         for (int i = 0; i < nteams; ++i)
@@ -1398,7 +1476,7 @@ DistributionMapping::makeSFC (const MultiFab& weight, bool sort)
 }
 
 std::vector<std::vector<int> >
-DistributionMapping::makeSFC (const BoxArray& ba)
+DistributionMapping::makeSFC (const BoxArray& ba, bool use_box_vol)
 {
     std::vector<SFCToken> tokens;
 
@@ -1408,10 +1486,13 @@ DistributionMapping::makeSFC (const BoxArray& ba)
 
     int maxijk = 0;
 
+    Real vol_sum = 0;
     for (int i = 0; i < N; ++i)
     {
 	const Box& bx = ba[i];
-        tokens.push_back(SFCToken(i,bx.smallEnd(),1.0));
+        const auto & bx_vol = (use_box_vol ? bx.volume() : 1);
+        tokens.push_back(SFCToken(i,bx.smallEnd(),bx_vol));
+        vol_sum += bx_vol;
 
         const SFCToken& token = tokens.back();
 
@@ -1433,7 +1514,8 @@ DistributionMapping::makeSFC (const BoxArray& ba)
     std::sort(tokens.begin(), tokens.end(), SFCToken::Compare());
 
     const int nprocs = ParallelContext::NProcsSub();
-    Real volper = static_cast<Real>(N)/static_cast<Real>(nprocs);
+    Real volper;
+    volper = vol_sum / nprocs;
 
     std::vector< std::vector<int> > r(nprocs);
     Distribute(tokens, nprocs, volper, r);
diff --git a/Src/Base/AMReX_Machine.H b/Src/Base/AMReX_Machine.H
new file mode 100644
index 00000000000..b20817128fd
--- /dev/null
+++ b/Src/Base/AMReX_Machine.H
@@ -0,0 +1,18 @@
+#ifndef AMREX_MACHINE_H
+#define AMREX_MACHINE_H
+
+#include <AMReX_Vector.H>
+
+namespace amrex {
+namespace machine {
+
+void Initialize (); // called in amrex::Initialize()
+void Finalize ();
+
+// find the best topologically close neighborhood of ranks
+// returns a vector of global or local rank IDs based on flag_local_ranks
+Vector<int> find_best_nbh (int rank_n, bool flag_local_ranks = false);
+
+}}
+
+#endif
diff --git a/Src/Base/AMReX_Machine.cpp b/Src/Base/AMReX_Machine.cpp
new file mode 100644
index 00000000000..f2dfac9edc6
--- /dev/null
+++ b/Src/Base/AMReX_Machine.cpp
@@ -0,0 +1,586 @@
+#include <string>
+#include <vector>
+#include <array>
+#include <fstream>
+#include <cassert>
+#include <sstream>
+#include <map>
+#include <unordered_map>
+
+#include <AMReX_Print.H>
+#include <AMReX_ParmParse.H>
+#include <AMReX_ParallelReduce.H>
+#include <AMReX_Utility.H>
+#include <AMReX_Machine.H>
+
+using namespace amrex;
+
+namespace {
+
+struct DoubleInt {
+    double d;
+    int i;
+};
+
+using Coord = Array<int, 4>;
+
+// returns coordinate in an index space with no switches
+// for dragonfly network
+Coord read_df_node_coord (const std::string & name)
+{
+    int cabx, caby, cab_chas, slot, node;
+    {
+        std::ifstream ifs {"/proc/cray_xt/cname"};
+        if (!ifs) {
+            // not on a cray
+            return Coord {0,0,0,0}; // initializer_list
+        }
+        char t0, t1, t2, t3, t4;
+        ifs >> t0 >> cabx >> t1 >> caby >> t2 >> cab_chas >> t3 >> slot >> t4 >> node;
+        assert(t0 == 'c' && t1 == '-' && t2 == 'c' && t3 == 's' && t4 == 'n');
+    }
+
+    int group = 0;
+    if (name == "edison") {
+        group = cabx / 2 + caby * 4; // 2 cabinets per group, 4 groups per row
+        if (group > 12) { group--; } // nominal "group 12" is missing
+    } else if (name == "cori") {
+        group = cabx / 2 + caby * 6; // 2 cabinets per group, 6 groups per row
+    } else {
+        Print() << "Could not determine group!";
+        std::abort();
+    }
+    int chas = cab_chas + 3*(cabx & 1); // 2 cabinets per group (6 chassis per group)
+
+    return Coord {node, slot, chas, group};
+}
+
+std::string get_mpi_processor_name ()
+{
+    std::string result;
+#if BL_USE_MPI
+    int len;
+    char name[MPI_MAX_PROCESSOR_NAME];
+    MPI_Get_processor_name(name, &len);
+    result = std::string(name);
+#endif
+    return result;
+}
+
+// assumes groups are in 4x16x6 configuration
+int df_coord_to_id (const Coord & c)
+{
+    return c[0] + 4 * (c[1] + 16 * (c[2] + 6 * c[3]));
+}
+
+// assumes groups are in 4x16x6 configuration
+Coord df_id_to_coord (int id)
+{
+    int node = id % 4;  id /= 4;
+    int slot = id % 16; id /= 16;
+    int chas = id % 6;  id /= 6;
+    int group = id;
+    return Coord {node, slot, chas, group};
+}
+
+template <class T, size_t N>
+std::string to_str(const Array<T, N> & a)
+{
+    std::ostringstream oss;
+    oss << "(";
+    bool first = true;
+    for (int i = 0; i < N; ++i) {
+        if (!first) oss << ",";
+        oss << a[i];
+        first = false;
+    }
+    oss << ")";
+    return oss.str();
+}
+
+template <class T>
+std::string to_str(const Vector<T> & v)
+{
+    std::ostringstream oss;
+    oss << "(";
+    bool first = true;
+    for (int i = 0; i < v.size(); ++i) {
+        if (!first) oss << ",";
+        oss << v[i];
+        first = false;
+    }
+    oss << ")";
+    return oss.str();
+}
+
+Vector<int> get_subgroup_ranks ()
+{
+    int rank_n = ParallelContext::NProcsSub();
+    Vector<int> lranks(rank_n);
+    for (int i = 0; i < rank_n; ++i) {
+        lranks[i] = i;
+    }
+
+    Vector<int> granks(rank_n);
+    ParallelContext::local_to_global_rank(granks.data(), lranks.data(), rank_n);
+    return granks;
+}
+
+int pair_n (int x) {
+    return x*(x-1)/2;
+}
+
+int df_dist (const Coord & a, const Coord & b)
+{
+    if (a[3] != b[3]) {
+        // large penalty for traversing across groups
+        return 20;
+    } else {
+        // same group
+        int slot_diff = (a[1] != b[1] ? 1 : 0);
+        int chas_diff = (a[2] != b[2] ? 1 : 0);
+        if (slot_diff + chas_diff == 0 && a[0] == b[0]) {
+            // same node
+            return 0;
+        } else {
+            // add 2 for first and last node-to-switch hops
+            return 2 + slot_diff + chas_diff;
+        }
+    }
+}
+
+Coord id_to_coord (int id)
+{
+    // TODO: implement support for other types of networks
+    return df_id_to_coord(id);
+}
+
+int dist (const Coord & a, const Coord & b)
+{
+    // TODO: implement support for other types of networks
+    return df_dist(a, b);
+}
+
+struct Candidate
+{
+    int id;
+    Coord coord;
+    // how many ranks on this node
+    int rank_n = 0;
+    // sum of pairwise rank distances from the candidate node to already chosen nodes
+    int sum_dist = 0;
+
+    Candidate () = default;
+    Candidate (int i) : id(i), coord(id_to_coord(id)) {}
+};
+
+class NeighborhoodCache
+{
+  public:
+    void add (uint64_t key, Vector<int> val) {
+        AMREX_ASSERT(cache.count(key) == 0);
+        cache[key] = std::move(val);
+    }
+    bool get (uint64_t key, Vector<int> & val) {
+        bool result = cache.count(key) > 0;
+        if (result) {
+            val = cache.at(key);
+        }
+        return result;
+    }
+
+    // result is dependent on both the current set of ranks
+    // and the size of the neighborhood desired
+    uint64_t hash (const Vector<int> & cur_ranks, int nbh_rank_n) {
+        auto result = hash_vector(cur_ranks);
+        hash_combine(result, nbh_rank_n);
+        return result;
+    }
+
+  private:
+    std::unordered_map<uint64_t, Vector<int>> cache;
+};
+
+class Machine
+{
+  public:
+    Machine () {
+        get_params();
+        get_machine_envs();
+        node_ids = get_node_ids();
+    }
+
+    // find a compact neighborhood of size rank_n in the current ParallelContext subgroup
+    Vector<int> find_best_nbh (int nbh_rank_n, bool flag_local_ranks)
+    {
+#ifdef BL_USE_MPI
+        BL_PROFILE("Machine::find_best_nbh()");
+
+        auto sg_g_ranks = get_subgroup_ranks();
+        auto sg_rank_n = sg_g_ranks.size();
+        if (flag_verbose) {
+            Print() << "Machine::find_best_nbh(): called for " << nbh_rank_n
+                    << " of " << sg_rank_n << " ranks" << std::endl;
+        }
+
+        Vector<int> result;
+        auto key = nbh_cache.hash(sg_g_ranks, nbh_rank_n);
+        if (nbh_cache.get(key, result)) {
+            if (flag_verbose) {
+                Print() << "Machine::find_best_nbh(): found neighborhood in cache" << std::endl;
+            }
+        } else {
+            // get node IDs of current subgroup
+            Vector<int> sg_node_ids(sg_rank_n);
+            std::unordered_map<int, std::vector<int>> node_ranks;
+            for (int i = 0; i < sg_rank_n; ++i) {
+                AMREX_ASSERT(sg_g_ranks[i] >= 0 && sg_g_ranks[i] < node_ids.size());
+                sg_node_ids[i] = node_ids[sg_g_ranks[i]];
+                if (flag_local_ranks) {
+                    node_ranks[sg_node_ids[i]].push_back(i);
+                } else {
+                    node_ranks[sg_node_ids[i]].push_back(sg_g_ranks[i]);
+                }
+            }
+
+            if (flag_very_verbose) {
+                Print() << "SubRank: GloRank: Node ID: Node Coord:" << std::endl;
+                for (int i = 0; i < sg_rank_n; ++i) {
+                    Print() << "  " << i << ": " << sg_g_ranks[i] << ": " << sg_node_ids[i]
+                            << ": " << to_str(id_to_coord(sg_node_ids[i])) << std::endl;
+                }
+            }
+
+            Vector<int> local_nbh;
+            double score;
+            auto rank_me = ParallelContext::MyProcSub();
+            tie(local_nbh, score) = search_local_nbh(rank_me, sg_node_ids, nbh_rank_n);
+
+            if (flag_verbose) {
+                Vector<int> base_nbh;
+                double base_score;
+                tie(base_nbh, base_score) = baseline_score(sg_node_ids, nbh_rank_n);
+
+                Print() << "Baseline neighborhood: " << to_str(base_nbh) << ", score = " << base_score << std::endl;
+                Print() << "Rank 0's neighborhood: " << to_str(local_nbh) << ", score = " << score << std::endl;
+            }
+
+            // determine the best neighborhood among ranks
+            DoubleInt my_score_with_id {score, rank_me}, min_score_with_id;
+            MPI_Allreduce(&my_score_with_id, &min_score_with_id, 1, MPI_DOUBLE_INT, MPI_MINLOC, ParallelContext::CommunicatorSub());
+            double winner_score = min_score_with_id.d;
+            int    winner_rank  = min_score_with_id.i;
+
+            // broadcast the best hood from winner rank to everyone
+            int local_nbh_size = local_nbh.size();
+            MPI_Bcast(&local_nbh_size, 1, MPI_INT, winner_rank, ParallelContext::CommunicatorSub());
+            local_nbh.resize(local_nbh_size);
+            MPI_Bcast(local_nbh.data(), local_nbh.size(), MPI_INT, winner_rank, ParallelContext::CommunicatorSub()); 
+
+            std::sort(local_nbh.begin(), local_nbh.end());
+            if (flag_verbose) {
+                Print() << "Winning neighborhood: " << winner_rank << ": " << to_str(local_nbh)
+                        << ", score = " << winner_score << std::endl;
+            }
+
+            result.reserve(nbh_rank_n);
+            for (int i = 0; i < local_nbh.size(); ++i) {
+                for (auto rank : node_ranks.at(local_nbh[i])) {
+                    if (result.size() < nbh_rank_n) {
+                        result.push_back(rank);
+                    }
+                }
+            }
+            nbh_cache.add(key, result);
+        }
+
+        if (flag_very_verbose) {
+            Print() << "Ranks in neighborhood: " << to_str(result) << std::endl;
+        }
+
+        return result;
+#else
+        return Vector<int>(nbh_rank_n, 0);
+#endif
+    }
+
+  private:
+
+    std::string hostname;
+    std::string nersc_host;
+    std::string partition;
+    std::string node_list;
+    std::string topo_addr;
+
+    int flag_verbose = 0;
+    int flag_very_verbose = 0;
+    bool flag_nersc_df;
+    int my_node_id;
+    Vector<int> node_ids;
+
+    NeighborhoodCache nbh_cache;
+
+    void get_params ()
+    {
+        ParmParse pp("machine");
+        pp.query("verbose", flag_verbose);
+        pp.query("very_verbose", flag_very_verbose);
+    }
+
+    std::string get_env_str (std::string env_key)
+    {
+        std::string result;
+        auto val_c_str = std::getenv(env_key.c_str());
+        if (val_c_str) {
+            result = std::string(val_c_str);
+        }
+        return result;
+    }
+
+    void get_machine_envs ()
+    {
+        hostname   = get_env_str("HOSTNAME");
+        nersc_host = get_env_str("NERSC_HOST");
+        flag_nersc_df = (nersc_host == "edison" ||
+                         nersc_host == "cori" ||
+                         nersc_host == "saul");
+
+        if (flag_nersc_df) {
+            partition  = get_env_str("SLURM_JOB_PARTITION");
+            node_list  = get_env_str("SLURM_NODELIST");
+            topo_addr  = get_env_str("SLURM_TOPOLOGY_ADDR");
+
+            if (flag_verbose) {
+                Print() << "HOSTNAME = " << hostname << std::endl;
+                Print() << "NERSC_HOST = " << nersc_host << std::endl;
+                Print() << "SLURM_JOB_PARTITION = " << partition << std::endl;
+                Print() << "SLURM_NODELIST = " << node_list << std::endl;
+                Print() << "SLURM_TOPOLOGY_ADDR = " << topo_addr << std::endl;
+            }
+        }
+    }
+
+    // get this rank's machine node ID
+    int get_my_node_id ()
+    {
+        int result = -1;
+        if (flag_nersc_df) {
+            std::string tag = "nid";
+            auto pos = topo_addr.find(tag);
+            if (pos != std::string::npos) {
+                result = stoi(topo_addr.substr(pos + tag.size())); // assumes format ".*nid(\d+)"
+                if (flag_verbose) {
+                    Print() << "Got node ID from SLURM_TOPOLOGY_ADDR: " << result << std::endl;
+                }
+#if BL_USE_MPI
+            } else {
+                auto mpi_proc_name = get_mpi_processor_name();
+                Print() << "MPI_Get_processor_name: " << mpi_proc_name << std::endl;
+                pos = mpi_proc_name.find(tag);
+                if (pos != std::string::npos) {
+                    result = stoi(mpi_proc_name.substr(pos + tag.size())); // assumes format ".*nid(\d+)"
+                    if (flag_verbose) {
+                        Print() << "Got node ID from MPI_Get_processor_name(): " << result << std::endl;
+                    }
+                }
+#endif
+            }
+
+            // check result
+            AMREX_ALWAYS_ASSERT(result != -1);
+#ifndef NDEBUG
+            auto coord = read_df_node_coord(nersc_host);
+            int id_from_coord = df_coord_to_id(coord);
+            AMREX_ASSERT(id_from_coord == result);
+#endif
+        } else {
+            result = 0;
+        }
+
+        return result;
+    }
+
+    // get all node IDs in this job, indexed by job rank
+    // this is collective over ALL ranks in the job
+    Vector<int> get_node_ids ()
+    {
+        int node_id = -1;
+        Vector<int> ids(ParallelDescriptor::NProcs(), 0);
+#ifdef BL_USE_MPI
+        node_id = get_my_node_id();
+        ParallelAllGather::AllGather(node_id, ids.data(), ParallelContext::CommunicatorAll());
+#endif
+        if (flag_verbose) {
+            std::map<int, Vector<int>> node_ranks;
+            for (int i = 0; i < ids.size(); ++i) {
+                node_ranks[ids[i]].push_back(i);
+            }
+            Print() << "Node ID: Node Coord: Ranks:" << std::endl;
+            for (const auto & p : node_ranks) {
+                Print() << "  " << p.first << ": " << to_str(id_to_coord(p.first))
+                        << ": " << to_str(p.second) << std::endl;
+            }
+        }
+        return ids;
+    }
+
+    // do a local search starting at current node
+    std::pair<Vector<int>, double>
+    baseline_score(const Vector<int> & sg_node_ids, int nbh_rank_n)
+    {
+        AMREX_ASSERT(sg_node_ids.size() > 0 && nbh_rank_n > 0 &&
+                     nbh_rank_n <= sg_node_ids.size());
+
+        // construct map of node candidates to select
+        std::map<int, Candidate> cand_map;
+        for (int i = 0; i < nbh_rank_n; ++i) {
+            auto node_id = sg_node_ids[i];
+            if (cand_map.count(node_id) == 0) {
+                cand_map[node_id] = Candidate(node_id);
+            }
+            cand_map.at(node_id).rank_n++;
+        }
+
+        Vector<int> result(cand_map.size());
+        Vector<Candidate> candidates(cand_map.size());
+        int idx = 0;
+        for (auto & p : cand_map) {
+            result[idx] = p.second.id;
+            candidates[idx++] = std::move(p.second);
+        }
+
+        int sum_dist = 0;
+        for (int j = 1; j < candidates.size(); ++j) {
+            const auto & b = candidates[j];
+            for (int i = 0; i < j; ++i) {
+                const auto & a = candidates[i];
+                auto pair_dist = dist(a.coord, b.coord);
+                // multiply distance by number of rank pairs across the two nodes
+                sum_dist += pair_dist * (a.rank_n * b.rank_n);
+                if (flag_very_verbose) {
+                    Print() << "    Distance from " << a.id
+                            << " to " << b.id
+                            << ": " << pair_dist << std::endl;
+                }
+            }
+        }
+        double score = (nbh_rank_n > 1) ? (static_cast<double>(sum_dist) / pair_n(nbh_rank_n)) : 0;
+        return std::make_pair(std::move(result), score);
+    }
+
+    // do a local search starting at current node
+    std::pair<Vector<int>, double>
+    search_local_nbh(int rank_me, const Vector<int> & sg_node_ids, int nbh_rank_n)
+    {
+        BL_PROFILE("Machine::search_local_nbh()");
+
+        Print() << "Machine::search_local_nbh() called ..." << std::endl;
+
+        Vector<int> result;
+
+        // construct map of node candidates to select
+        std::map<int, Candidate> candidates;
+        for (auto node_id : sg_node_ids) {
+            if (candidates.count(node_id) == 0) {
+                candidates[node_id] = Candidate(node_id);
+            }
+            candidates.at(node_id).rank_n++;
+        }
+
+        if (flag_very_verbose) {
+            Print() << "  Candidates:" << std::endl;
+            for (const auto & p : candidates) {
+                const auto & cand = p.second;
+                Print() << "    " << cand.id << " : " << to_str(cand.coord)
+                        << ": " << cand.rank_n << " ranks" << std::endl;
+            }
+        }
+
+        AMREX_ASSERT(rank_me >= 0 && rank_me < sg_node_ids.size());
+        Candidate cur_node = std::move(candidates.at(sg_node_ids[rank_me]));
+        candidates.erase(cur_node.id);
+
+        // add source_node
+        result.push_back(cur_node.id);
+        int total_rank_n = cur_node.rank_n;
+        int total_pairs_dist = 0;
+        if (flag_verbose) {
+            Print() << "  Added " << cur_node.id
+                    << ": " << to_str(cur_node.coord)
+                    << ", ranks: " << cur_node.rank_n
+                    << ", total ranks: " << total_rank_n
+                    << ", avg dist: " << 0 << std::endl;
+        }
+        if (total_rank_n >= nbh_rank_n) {
+            return {std::move(result), 0};
+        }
+
+        double min_avg_dist;
+        while (total_rank_n < nbh_rank_n)
+        {
+            min_avg_dist = std::numeric_limits<double>::max();
+            Candidate * next_node = nullptr;
+            // update candidates with their pairwise rank distances to cur_node
+            for (auto & p : candidates) {
+                Candidate & cand_node = p.second;
+                auto cand_dist = dist(cand_node.coord, cur_node.coord);
+                // multiply distance by number of rank pairs across the two nodes
+                cand_node.sum_dist += cand_dist * (cand_node.rank_n * cur_node.rank_n);
+                double avg_dist = static_cast<double>(cand_node.sum_dist + total_pairs_dist) /
+                                  pair_n(cand_node.rank_n + total_rank_n);
+                if (flag_very_verbose) {
+                    Print() << "    Distance from " << cand_node.id
+                            << " to " << cur_node.id
+                            << ": " << cand_dist
+                            << ", candidate avg: " << avg_dist << std::endl;
+                }
+                // keep track of what should be the next node to add
+                if (avg_dist < min_avg_dist) {
+                    next_node = &cand_node;
+                    min_avg_dist = avg_dist;
+                }
+            }
+            cur_node = std::move(*next_node);
+            next_node = nullptr;
+            candidates.erase(cur_node.id);
+
+            // add cur_node to result
+            result.push_back(cur_node.id);
+            total_rank_n += cur_node.rank_n;
+            total_pairs_dist += cur_node.sum_dist;
+
+            if (flag_verbose) {
+                Print() << "  Added " << cur_node.id
+                        << ": " << to_str(cur_node.coord)
+                        << ", ranks: " << cur_node.rank_n
+                        << ", total ranks: " << total_rank_n
+                        << ", avg dist: " << min_avg_dist << std::endl;
+            }
+        }
+
+        return std::make_pair(std::move(result), min_avg_dist);
+    }
+};
+
+std::unique_ptr<Machine> the_machine;
+
+}
+
+namespace amrex {
+namespace machine {
+
+void Initialize () {
+    the_machine.reset(new Machine());
+    amrex::ExecOnFinalize(machine::Finalize);
+}
+
+void Finalize () {
+    the_machine.reset();
+}
+
+Vector<int> find_best_nbh (int rank_n, bool flag_local_ranks) {
+    AMREX_ASSERT(the_machine);
+    return the_machine->find_best_nbh(rank_n, flag_local_ranks);
+}
+
+}}
diff --git a/Src/Base/AMReX_MultiFabUtil.cpp b/Src/Base/AMReX_MultiFabUtil.cpp
index 295a1bc0b2a..a11d65457f4 100644
--- a/Src/Base/AMReX_MultiFabUtil.cpp
+++ b/Src/Base/AMReX_MultiFabUtil.cpp
@@ -238,6 +238,7 @@ namespace amrex
 		       const Geometry& fgeom, const Geometry& cgeom,
                        int scomp, int ncomp, const IntVect& ratio)
     {
+        BL_PROFILE("amrex::average_down_w_geom");
 
         if (S_fine.is_nodal() || S_crse.is_nodal())
         {
@@ -337,6 +338,7 @@ namespace amrex
     void average_down (const MultiFab& S_fine, MultiFab& S_crse,
                        int scomp, int ncomp, const IntVect& ratio)
     {
+        BL_PROFILE("amrex::average_down");
         AMREX_ASSERT(S_crse.nComp() == S_fine.nComp());
         AMREX_ASSERT((S_crse.is_cell_centered() && S_fine.is_cell_centered()) ||
                      (S_crse.is_nodal()         && S_fine.is_nodal()));
diff --git a/Src/Base/AMReX_ParallelContext.H b/Src/Base/AMReX_ParallelContext.H
index c71fc0d82b6..c9f7f59de23 100644
--- a/Src/Base/AMReX_ParallelContext.H
+++ b/Src/Base/AMReX_ParallelContext.H
@@ -58,6 +58,8 @@ inline int NProcsAll () { return frames[0].NProcs(); }
 inline int MyProcAll () { return frames[0].MyProc(); }
 // IO rank in world communicator
 inline int IOProcessorNumberAll () { return frames[0].IOProc(); }
+// Am IO processor for world communicator?
+inline bool IOProcessorAll () { return MyProcAll() == IOProcessorNumberAll(); }
 // Pointer to ofstream
 inline std::ofstream * OFSPtrAll () { return frames[0].get_ofs_ptr(); }
 
@@ -71,6 +73,8 @@ inline int NProcsSub () { return frames.back().NProcs(); }
 inline int MyProcSub () { return frames.back().MyProc(); }
 // IO sub-rank in current frame
 inline int IOProcessorNumberSub () { return frames.back().IOProc(); }
+// Am IO processor for current frame?
+inline bool IOProcessorSub () { return MyProcSub() == IOProcessorNumberSub(); }
 // Pointer to ofstream
 inline std::ofstream * OFSPtrSub () { return frames.back().get_ofs_ptr(); }
 
diff --git a/Src/Base/AMReX_ParallelReduce.H b/Src/Base/AMReX_ParallelReduce.H
index 0776f5875a1..5ae621bf060 100644
--- a/Src/Base/AMReX_ParallelReduce.H
+++ b/Src/Base/AMReX_ParallelReduce.H
@@ -36,9 +36,11 @@ namespace detail {
         auto mpi_op = mpi_ops[static_cast<int>(op)];
         Vector<T> tmp(v, v+cnt);
         if (root == -1) {
+            // TODO: add BL_COMM_PROFILE commands
             MPI_Allreduce(tmp.data(), v, cnt, ParallelDescriptor::Mpi_typemap<T>::type(),
                           mpi_op, comm);
         } else {
+            // TODO: add BL_COMM_PROFILE commands
             MPI_Reduce(tmp.data(), v, cnt, ParallelDescriptor::Mpi_typemap<T>::type(),
                        mpi_op, root, comm);
         }
@@ -65,9 +67,15 @@ namespace detail {
     {
         auto mpi_type = ParallelDescriptor::Mpi_typemap<T>::type();
         if (root == -1) {
+            // TODO: check these BL_COMM_PROFILE commands
+            BL_COMM_PROFILE(BLProfiler::Allgather, sizeof(T), BLProfiler::BeforeCall(),
+                            BLProfiler::NoTag());
             // const_cast for MPI-2
             MPI_Allgather(const_cast<T*>(v), cnt, mpi_type, vs, cnt, mpi_type, comm);
+            BL_COMM_PROFILE(BLProfiler::Allgather, sizeof(T), BLProfiler::AfterCall(),
+                            BLProfiler::NoTag());
         } else {
+            // TODO: add BL_COMM_PROFILE commands
             MPI_Gather(const_cast<T*>(v), cnt, mpi_type, vs, cnt, mpi_type, root, comm);
         }
     }
diff --git a/Src/Base/AMReX_Utility.H b/Src/Base/AMReX_Utility.H
index ba09a6cc17f..7d259f51159 100644
--- a/Src/Base/AMReX_Utility.H
+++ b/Src/Base/AMReX_Utility.H
@@ -318,6 +318,9 @@ namespace amrex
                                                std::chrono::steady_clock>::type;
     double second ();
 
+    template<typename T> void hash_combine (uint64_t & seed, const T & val);
+    template<typename T> uint64_t hash_vector (const Vector<T> & vec, uint64_t seed = 0xDEADBEEFDEADBEEF);
+
     template <class T>
     AMREX_GPU_HOST_DEVICE
     AMREX_INLINE const T& min (const T& a, const T& b)
@@ -447,4 +450,47 @@ extern "C" {
     void amrex_free (void* p);
 }
 
+// hash combiner borrowed from Boost
+/*
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+*/
+template<typename T>
+void
+amrex::hash_combine (uint64_t & seed, const T & val)
+{
+    seed ^= std::hash<T>()(val) + 0x9e3779b9 + (seed<<6) + (seed>>2);
+}
+
+template<typename T>
+uint64_t
+amrex::hash_vector (const Vector<T> & vec, uint64_t seed)
+{
+    for (const auto & x: vec) {
+        hash_combine(seed, x);
+    }
+    return seed;
+}
+
 #endif /*BL_UTILITY_H*/
diff --git a/Src/Base/CMakeLists.txt b/Src/Base/CMakeLists.txt
index 7a24d131f45..694bf0fa394 100644
--- a/Src/Base/CMakeLists.txt
+++ b/Src/Base/CMakeLists.txt
@@ -128,6 +128,8 @@ add_sources( AMReX_io_mod.F90 )
 
 add_sources( AMReX_BLProfiler.cpp AMReX_BLBackTrace.cpp )
 
+# Machine model
+add_sources ( AMReX_Machine.H AMReX_Machine.cpp )
 
 # LAZY mode not supported in CMake yet
 # ifeq ($(LAZY),TRUE)
diff --git a/Src/Base/Make.package b/Src/Base/Make.package
index 361adffbb65..cf512bb2a0e 100644
--- a/Src/Base/Make.package
+++ b/Src/Base/Make.package
@@ -211,6 +211,9 @@ ifeq ($(TINY_PROFILE),TRUE)
   C$(AMREX_BASE)_sources += AMReX_TinyProfiler.cpp
 endif
 
+# Machine model
+CEXE_headers += AMReX_Machine.H
+CEXE_sources += AMReX_Machine.cpp
 
 VPATH_LOCATIONS += $(AMREX_HOME)/Src/Base
 INCLUDE_LOCATIONS += $(AMREX_HOME)/Src/Base
diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.cpp b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.cpp
index ef74c5c1807..29b85ee7187 100644
--- a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.cpp
+++ b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.cpp
@@ -80,7 +80,7 @@ MLCGSolver::solve_bicgstab (MultiFab&       sol,
                             Real            eps_rel,
                             Real            eps_abs)
 {
-    BL_PROFILE_REGION("MLCGSolver::bicgstab");
+    BL_PROFILE("MLCGSolver::bicgstab");
 
     const int nghost = sol.nGrow(), ncomp = sol.nComp();
 
@@ -192,7 +192,9 @@ MLCGSolver::solve_bicgstab (MultiFab&       sol,
         //
         Real tvals[2] = { dotxy(t,t,true), dotxy(t,s,true) };
 
+        BL_PROFILE_VAR("MLCGSolver::ParallelAllReduce", blp_par);
         ParallelAllReduce::Sum(tvals,2,Lp.BottomCommunicator());
+        BL_PROFILE_VAR_STOP(blp_par);
 
         if ( tvals[0] )
 	{
@@ -260,7 +262,7 @@ MLCGSolver::solve_cg (MultiFab&       sol,
                       Real            eps_rel,
                       Real            eps_abs)
 {
-    BL_PROFILE_REGION("MLCGSolver::cg");
+    BL_PROFILE("MLCGSolver::cg");
 
     const int nghost = sol.nGrow(), ncomp = sol.nComp();
 
@@ -390,7 +392,11 @@ MLCGSolver::solve_cg (MultiFab&       sol,
 Real
 MLCGSolver::dotxy (const MultiFab& r, const MultiFab& z, bool local)
 {
-    return Lp.xdoty(amrlev, mglev, r, z, local);
+    BL_PROFILE_VAR_NS("MLCGSolver::ParallelAllReduce", blp_par);
+    if (!local) { BL_PROFILE_VAR_START(blp_par); }
+    Real result = Lp.xdoty(amrlev, mglev, r, z, local);
+    if (!local) { BL_PROFILE_VAR_STOP(blp_par); }
+    return result;
 }
 
 Real
@@ -402,6 +408,7 @@ MLCGSolver::norm_inf (const MultiFab& res, bool local)
       result = std::max(result,res.norm0(n,0,true));
 
     if (!local) {
+        BL_PROFILE("MLCGSolver::ParallelAllReduce");
         ParallelAllReduce::Max(result, Lp.BottomCommunicator());
     }
     return result;
diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H
index 360aaf5a344..02824fe284a 100644
--- a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H
+++ b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H
@@ -34,6 +34,62 @@ public:
     }
     virtual void update () override;
 
+#ifdef AMREX_SOFT_PERF_COUNTERS
+    struct Counters
+    {
+        size_t smooth_n = 0;
+        size_t smooth_cell_n = 0;
+        size_t apply_n = 0;
+        size_t apply_cell_n = 0;
+        size_t restrict_n = 0;
+        size_t restrict_cell_n = 0;
+        size_t interp_n = 0;
+        size_t interp_cell_n = 0;
+
+        void reset () {
+            smooth_n = 0;
+            smooth_cell_n = 0;
+            apply_n = 0;
+            apply_cell_n = 0;
+            restrict_n = 0;
+            restrict_cell_n = 0;
+            interp_n = 0;
+            interp_cell_n = 0;
+        }
+
+        void smooth (const MultiFab & mf) {
+            smooth_n++;
+            smooth_cell_n += mf.boxArray().numPts();
+        }
+        void apply (const MultiFab & mf) {
+            apply_n++;
+            apply_cell_n += mf.boxArray().numPts();
+        }
+        void restrict (const MultiFab & mf) {
+            restrict_n++;
+            restrict_cell_n += mf.boxArray().numPts();
+        }
+        void interpolate (const MultiFab & mf) {
+            interp_n++;
+            interp_cell_n += mf.boxArray().numPts();
+        }
+
+        void print () {
+            Print() << ParallelContext::MyProcSub() << ": perf: MLCellLinOp::smooth()        called " << smooth_n
+                    << " times on " << smooth_cell_n << " cells total" << std::endl;
+            Print() << ParallelContext::MyProcSub() << ": perf: MLCellLinOp::apply()         called " << apply_n
+                    << " times on " << apply_cell_n << " cells total" << std::endl;
+            Print() << ParallelContext::MyProcSub() << ": perf: MLCellLinOp::restriction()   called " << restrict_n
+                    << " times on " << restrict_cell_n << " cells total" << std::endl;
+            Print() << ParallelContext::MyProcSub() << ": perf: MLCellLinOp::interpolation() called " << interp_n
+                    << " times on " << interp_cell_n << " cells total" << std::endl;
+        }
+    };
+
+    // software counters for performance analysis
+    static Counters perf_counters;
+#endif
+
 protected:
 
     virtual bool isCrossStencil () const { return true; }
diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.cpp b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.cpp
index d47e7de0870..385e032ef3f 100644
--- a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.cpp
+++ b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.cpp
@@ -296,12 +296,18 @@ void
 MLCellLinOp::restriction (int, int, MultiFab& crse, MultiFab& fine) const
 {
     const int ncomp = getNComp();
+#ifdef AMREX_SOFT_PERF_COUNTERS
+    perf_counters.restrict(crse);
+#endif
     amrex::average_down(fine, crse, 0, ncomp, 2);
 }
 
 void
 MLCellLinOp::interpolation (int amrlev, int fmglev, MultiFab& fine, const MultiFab& crse) const
 {
+#ifdef AMREX_SOFT_PERF_COUNTERS
+    perf_counters.interpolate(fine);
+#endif
 #ifdef _OPENMP
 #pragma omp parallel
 #endif
@@ -336,6 +342,9 @@ MLCellLinOp::apply (int amrlev, int mglev, MultiFab& out, MultiFab& in, BCMode b
 {
     BL_PROFILE("MLCellLinOp::apply()");
     applyBC(amrlev, mglev, in, bc_mode, s_mode, bndry);
+#ifdef AMREX_SOFT_PERF_COUNTERS
+    perf_counters.apply(out);
+#endif
     Fapply(amrlev, mglev, out, in);
 }
 
@@ -348,6 +357,9 @@ MLCellLinOp::smooth (int amrlev, int mglev, MultiFab& sol, const MultiFab& rhs,
     {
         applyBC(amrlev, mglev, sol, BCMode::Homogeneous, StateMode::Solution,
                 nullptr, skip_fillboundary);
+#ifdef AMREX_SOFT_PERF_COUNTERS
+        perf_counters.smooth(sol);
+#endif
         Fsmooth(amrlev, mglev, sol, rhs, redblack);
         skip_fillboundary = false;
     }
@@ -843,4 +855,9 @@ MLCellLinOp::update ()
     if (MLLinOp::needsUpdate()) MLLinOp::update();
 }
 
+#ifdef AMREX_SOFT_PERF_COUNTERS
+// perf_counters
+MLCellLinOp::Counters MLCellLinOp::perf_counters;
+#endif
+
 }
diff --git a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H
index 32749de0505..52a87c675f5 100644
--- a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H
+++ b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H
@@ -56,6 +56,9 @@ public:
 
     enum struct Location { FaceCenter, FaceCentroid, CellCenter, CellCentroid };
 
+    static void Initialize ();
+    static void Finalize ();
+
     MLLinOp ();
     virtual ~MLLinOp ();
 
@@ -279,6 +282,7 @@ private:
     static void makeConsolidatedDMap (const Vector<BoxArray>& ba, Vector<DistributionMapping>& dm,
                                       int ratio, int strategy);
     MPI_Comm makeSubCommunicator (const DistributionMapping& dm);
+    void remapNeighborhoods (Vector<DistributionMapping> & dms);
 };
 
 }
diff --git a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.cpp b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.cpp
index 1a08ce6caef..131ef7b8b29 100644
--- a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.cpp
+++ b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.cpp
@@ -1,8 +1,13 @@
 
 #include <cmath>
 #include <algorithm>
+#include <unordered_map>
+#include <set>
+#include <AMReX_Utility.H>
 #include <AMReX_MLLinOp.H>
+#include <AMReX_MLCellLinOp.H>
 #include <AMReX_ParmParse.H>
+#include <AMReX_Machine.H>
 
 #ifdef AMREX_USE_EB
 #include <AMReX_EB2.H>
@@ -21,9 +26,95 @@ constexpr int MLLinOp::mg_box_min_width;
 
 namespace {
     // experimental features
-    bool initialized = false;
+    bool initialized = false; // track initialization of static state
+    int consolidation_threshold = -1;
     int consolidation_ratio = 2;
     int consolidation_strategy = 3;
+
+    int flag_verbose_linop = 0;
+    int flag_comm_cache = 0;
+    int flag_use_mota = 0;
+    int remap_nbh_lb = 1;
+
+#ifdef BL_USE_MPI
+    class CommCache
+    {
+      public:
+        CommCache () = default;
+        CommCache (const CommCache&) = delete;
+        CommCache (CommCache&&) = delete;
+        void operator= (const CommCache&) = delete;
+        void operator= (CommCache&&) = delete;
+
+        ~CommCache () {
+            for (auto & p : cache) {
+                if (p.second != MPI_COMM_NULL) {
+                    MPI_Comm_free(&p.second);
+                }
+            }
+        }
+        void add (size_t key, MPI_Comm comm) {
+            AMREX_ASSERT(cache.count(key) == 0);
+            cache[key] = comm;
+        }
+        bool get (size_t key, MPI_Comm &comm) {
+            bool result = cache.count(key) > 0;
+            if (result) {
+                comm = cache.at(key);
+            }
+            return result;
+        }
+
+      private:
+        std::unordered_map<size_t, MPI_Comm> cache;
+    };
+
+    std::unique_ptr<CommCache> comm_cache;
+#endif
+
+    Vector<int> get_subgroup_ranks ()
+    {
+        int rank_n = ParallelContext::NProcsSub();
+        Vector<int> lranks(rank_n);
+        for (int i = 0; i < rank_n; ++i) {
+            lranks[i] = i;
+        }
+
+        Vector<int> granks(rank_n);
+        ParallelContext::local_to_global_rank(granks.data(), lranks.data(), rank_n);
+        return granks;
+    }
+}
+
+// static member function
+void MLLinOp::Initialize ()
+{
+    ParmParse pp("mg");
+    pp.query("consolidation_threshold", consolidation_threshold);
+    pp.query("consolidation_ratio", consolidation_ratio);
+    pp.query("consolidation_strategy", consolidation_strategy);
+    pp.query("verbose_linop", flag_verbose_linop);
+    pp.query("comm_cache", flag_comm_cache);
+    pp.query("mota", flag_use_mota);
+    pp.query("remap_nbh_lb", remap_nbh_lb);
+
+#ifdef BL_USE_MPI
+    comm_cache.reset(new CommCache());
+#endif
+    amrex::ExecOnFinalize(MLLinOp::Finalize);
+    initialized = true;
+}
+
+// static member function
+void MLLinOp::Finalize ()
+{
+    initialized = false;
+#ifdef BL_USE_MPI
+    comm_cache.reset();
+#endif
+#ifdef AMREX_SOFT_PERF_COUNTERS
+    MLCellLinOp::perf_counters.reset();
+#endif
 }
 
 MLLinOp::MLLinOp () {}
@@ -40,10 +131,7 @@ MLLinOp::define (const Vector<Geometry>& a_geom,
     BL_PROFILE("MLLinOp::define()");
 
     if (!initialized) {
-	ParmParse pp("mg");
-	pp.query("consolidation_ratio", consolidation_ratio);
-	pp.query("consolidation_strategy", consolidation_strategy);
-	initialized = true;
+        Initialize();
     }
 
     info = a_info;
@@ -158,6 +246,7 @@ MLLinOp::defineGrids (const Vector<Geometry>& a_geom,
 
     bool agged = false;
     bool coned = false;
+    int agg_lev, con_lev;
 
     if (info.do_agglomeration && aggable)
     {
@@ -237,6 +326,7 @@ MLLinOp::defineGrids (const Vector<Geometry>& a_geom,
 
                 m_dmap[0].push_back(DistributionMapping());
                 agged = true;
+                agg_lev = last_coarsenableto_lev;
             }
 
             m_num_mg_levels[0] = m_grids[0].size();
@@ -245,12 +335,14 @@ MLLinOp::defineGrids (const Vector<Geometry>& a_geom,
     else
     {
         int rr = mg_coarsen_ratio;
-        Real avg_npts, threshold_npts;
+        Real avg_npts;
         if (info.do_consolidation) {
             avg_npts = static_cast<Real>(a_grids[0].d_numPts()) / static_cast<Real>(ParallelContext::NProcsSub());
-            threshold_npts = static_cast<Real>(AMREX_D_TERM(info.con_grid_size,
-                                                            *info.con_grid_size,
-                                                            *info.con_grid_size));
+            if (consolidation_threshold == -1) {
+                consolidation_threshold = static_cast<Real>(AMREX_D_TERM(info.con_grid_size,
+                                                                         *info.con_grid_size,
+                                                                         *info.con_grid_size));
+            }
         }
         while (m_num_mg_levels[0] < info.max_coarsening_level + 1
                and a_geom[0].Domain().coarsenable(rr)
@@ -263,9 +355,10 @@ MLLinOp::defineGrids (const Vector<Geometry>& a_geom,
 
             if (info.do_consolidation)
             {
-                if (avg_npts/(AMREX_D_TERM(rr,*rr,*rr)) < 0.999*threshold_npts)
+                if (avg_npts/(AMREX_D_TERM(rr,*rr,*rr)) < 0.999*consolidation_threshold)
                 {
                     coned = true;
+                    con_lev = m_dmap[0].size();
                     m_dmap[0].push_back(DistributionMapping());
                 }
                 else
@@ -292,6 +385,11 @@ MLLinOp::defineGrids (const Vector<Geometry>& a_geom,
         makeConsolidatedDMap(m_grids[0], m_dmap[0], consolidation_ratio, consolidation_strategy);
     }
 
+    if (flag_use_mota && (agged || coned))
+    {
+        remapNeighborhoods(m_dmap[0]);
+    }
+
     if (info.do_agglomeration || info.do_consolidation)
     {
         m_bottom_comm = makeSubCommunicator(m_dmap[0].back());
@@ -304,6 +402,19 @@ MLLinOp::defineGrids (const Vector<Geometry>& a_geom,
     m_do_agglomeration = agged;
     m_do_consolidation = coned;
 
+    if (flag_verbose_linop) {
+        if (agged) {
+            Print() << "MLLinOp::defineGrids(): agglomerated AMR level 0 starting at MG level "
+                    << agg_lev << " of " << m_num_mg_levels[0] << std::endl;
+        } else if (coned) {
+            Print() << "MLLinOp::defineGrids(): consolidated AMR level 0 starting at MG level "
+                    << con_lev << " of " << m_num_mg_levels[0]
+                    << " (ratio = " << consolidation_ratio << ")" << std::endl;
+        } else {
+            Print() << "MLLinOp::defineGrids(): no agglomeration or consolidation of AMR level 0" << std::endl;
+        }
+    }
+
     for (int amrlev = 0; amrlev < m_num_amr_levels; ++amrlev)
     {
         for (int mglev = 1; mglev < m_num_mg_levels[amrlev]; ++mglev)
@@ -377,30 +488,59 @@ MLLinOp::makeSubCommunicator (const DistributionMapping& dm)
     BL_PROFILE("MLLinOp::makeSubCommunicator()");
 
 #ifdef BL_USE_MPI
-    MPI_Comm newcomm;
-    MPI_Group defgrp, newgrp;
-
-    MPI_Comm_group(m_default_comm, &defgrp);
 
     Vector<int> newgrp_ranks = dm.ProcessorMap();
     std::sort(newgrp_ranks.begin(), newgrp_ranks.end());
     auto last = std::unique(newgrp_ranks.begin(), newgrp_ranks.end());
     newgrp_ranks.erase(last, newgrp_ranks.end());
     
-    if (ParallelContext::CommunicatorSub() == ParallelDescriptor::Communicator()) {
-        MPI_Group_incl(defgrp, newgrp_ranks.size(), newgrp_ranks.data(), &newgrp);
-    } else {
-        Vector<int> local_newgrp_ranks(newgrp_ranks.size());
-        ParallelContext::global_to_local_rank(local_newgrp_ranks.data(),
-                                              newgrp_ranks.data(), newgrp_ranks.size());
-        MPI_Group_incl(defgrp, local_newgrp_ranks.size(), local_newgrp_ranks.data(), &newgrp);
+    if (flag_verbose_linop) {
+        Print() << "MLLinOp::makeSubCommunicator(): called for " << newgrp_ranks.size() << " ranks" << std::endl;
     }
 
-    MPI_Comm_create(m_default_comm, newgrp, &newcomm);   
-    m_raii_comm.reset(new CommContainer(newcomm));
+    MPI_Comm newcomm;
+    bool cache_hit = false;
+    uint64_t key = 0;
+    if (flag_comm_cache) {
+        AMREX_ASSERT(comm_cache);
+        key = hash_vector(newgrp_ranks, hash_vector(get_subgroup_ranks()));
+        cache_hit = comm_cache->get(key, newcomm);
+        if (cache_hit && flag_verbose_linop) {
+            Print() << "MLLinOp::makeSubCommunicator(): found subcomm in cache" << std::endl;
+        }
+    }
+
+    if (!flag_comm_cache || !cache_hit) {
+        MPI_Group defgrp, newgrp;
+        MPI_Comm_group(m_default_comm, &defgrp);
+        if (ParallelContext::CommunicatorSub() == ParallelDescriptor::Communicator()) {
+            MPI_Group_incl(defgrp, newgrp_ranks.size(), newgrp_ranks.data(), &newgrp);
+        } else {
+            Vector<int> local_newgrp_ranks(newgrp_ranks.size());
+            ParallelContext::global_to_local_rank(local_newgrp_ranks.data(),
+                                                  newgrp_ranks.data(), newgrp_ranks.size());
+            MPI_Group_incl(defgrp, local_newgrp_ranks.size(), local_newgrp_ranks.data(), &newgrp);
+        }
+
+        if (flag_verbose_linop) {
+            Print() << "MLLinOp::makeSubCommunicator(): MPI_Comm_create: (";
+            for (auto rank : newgrp_ranks) {
+                Print() << rank << ",";
+            }
+            Print() << "\b)" << std::endl;
+        }
+
+        MPI_Comm_create(m_default_comm, newgrp, &newcomm);
+
+        if (flag_comm_cache) {
+            comm_cache->add(key, newcomm);
+        } else {
+            m_raii_comm.reset(new CommContainer(newcomm));
+        }
 
-    MPI_Group_free(&defgrp);
-    MPI_Group_free(&newgrp);
+        MPI_Group_free(&defgrp);
+        MPI_Group_free(&newgrp);
+    }
 
     return newcomm;
 #else
@@ -490,6 +630,44 @@ MLLinOp::makeConsolidatedDMap (const Vector<BoxArray>& ba, Vector<DistributionMa
     }
 }
 
+void
+MLLinOp::remapNeighborhoods (Vector<DistributionMapping> & dms)
+{
+    BL_PROFILE("MLLinOp::remapNeighborhoods()");
+
+    if (flag_verbose_linop) {
+        Print() << "Remapping ranks to neighborhoods ..." << std::endl;
+    }
+
+    for (int j = 1; j < dms.size(); ++j)
+    {
+        const Vector<int> & pmap = dms[j].ProcessorMap();
+        std::set<int> g_ranks_set(pmap.begin(), pmap.end());
+        auto lev_rank_n = g_ranks_set.size();
+        if (lev_rank_n >= remap_nbh_lb && lev_rank_n < ParallelContext::NProcsSub())
+        {
+            // find best neighborhood with lev_rank_n ranks
+            auto nbh_g_ranks = machine::find_best_nbh(lev_rank_n);
+            AMREX_ASSERT(nbh_g_ranks.size() == lev_rank_n);
+
+            // construct mapping from original global rank to neighborhood global rank
+            int idx = 0;
+            std::unordered_map<int, int> rank_mapping;
+            for (auto orig_g_rank : g_ranks_set) {
+                AMREX_ASSERT(idx < nbh_g_ranks.size());
+                rank_mapping[orig_g_rank] = nbh_g_ranks[idx++];
+            }
+
+            // remap and create new DM
+            Vector<int> nbh_pmap(pmap.size());
+            for (int i = 0; i < pmap.size(); ++i) {
+                nbh_pmap[i] = rank_mapping.at(pmap[i]);
+            }
+            dms[j] = DistributionMapping(std::move(nbh_pmap));
+        }
+    }
+}
+
 #ifdef AMREX_USE_PETSC
 std::unique_ptr<PETScABecLap>
 MLLinOp::makePETSc () const
diff --git a/Src/LinearSolvers/MLMG/AMReX_MLMG.cpp b/Src/LinearSolvers/MLMG/AMReX_MLMG.cpp
index 55a791a650f..dfaff5d0e6b 100644
--- a/Src/LinearSolvers/MLMG/AMReX_MLMG.cpp
+++ b/Src/LinearSolvers/MLMG/AMReX_MLMG.cpp
@@ -53,7 +53,6 @@ Real
 MLMG::solve (const Vector<MultiFab*>& a_sol, const Vector<MultiFab const*>& a_rhs,
              Real a_tol_rel, Real a_tol_abs)
 {
-    BL_PROFILE_REGION("MLMG::solve()");
     BL_PROFILE("MLMG::solve()");
 
     if (bottom_solver == BottomSolver::hypre) {
@@ -357,21 +356,38 @@ MLMG::miniCycle (int amrlev)
     mgVcycle(amrlev, mglev);
 }
 
+namespace {
+
+void make_str_helper (std::ostringstream & oss) { }
+
+template <class T, class... Ts>
+void make_str_helper (std::ostringstream & oss, T x, Ts... xs) {
+    oss << x;
+    make_str_helper(oss, xs...);
+}
+
+template <class... Ts>
+std::string make_str (Ts... xs) {
+    std::ostringstream oss;
+    make_str_helper(oss, xs...);
+    return oss.str();
+}
+
+}
+
 // in   : Residual (res) 
 // out  : Correction (cor) from bottom to this function's local top
 void
 MLMG::mgVcycle (int amrlev, int mglev_top)
 {
     BL_PROFILE("MLMG::mgVcycle()");
-    BL_PROFILE_VAR_NS("MLMG::mgVcycle_up", blp_up);
-    BL_PROFILE_VAR_NS("MLMG::mgVcycle_bottom", blp_bottom);
-    BL_PROFILE_VAR_NS("MLMG::mgVcycle_down", blp_down);
 
     const int mglev_bottom = linop.NMGLevels(amrlev) - 1;
 
-    BL_PROFILE_VAR_START(blp_down);
     for (int mglev = mglev_top; mglev < mglev_bottom; ++mglev)
     {
+        std::string blp_mgv_down_lev_str = make_str("MLMG::mgVcycle_down::", mglev);
+        BL_PROFILE_VAR(blp_mgv_down_lev_str, blp_mgv_down_lev);
 
         if (verbose >= 4)
         {
@@ -402,9 +418,8 @@ MLMG::mgVcycle (int amrlev, int mglev_top)
         linop.restriction(amrlev, mglev+1, res[amrlev][mglev+1], rescor[amrlev][mglev]);
 
     }
-    BL_PROFILE_VAR_STOP(blp_down);
 
-    BL_PROFILE_VAR_START(blp_bottom);
+    BL_PROFILE_VAR("MLMG::mgVcycle_bottom", blp_bottom);
     if (amrlev == 0)
     {
         if (verbose >= 4)
@@ -447,9 +462,10 @@ MLMG::mgVcycle (int amrlev, int mglev_top)
     }
     BL_PROFILE_VAR_STOP(blp_bottom);
 
-    BL_PROFILE_VAR_START(blp_up);
     for (int mglev = mglev_bottom-1; mglev >= mglev_top; --mglev)
     {
+        std::string blp_mgv_up_lev_str = make_str("MLMG::mgVcycle_up::", mglev);
+        BL_PROFILE_VAR(blp_mgv_up_lev_str, blp_mgv_up_lev);
         // cor_fine += I(cor_crse)
         addInterpCorrection(amrlev, mglev);
         if (verbose >= 4)
@@ -470,7 +486,6 @@ MLMG::mgVcycle (int amrlev, int mglev_top)
                            << "   UP: Norm after  smooth " << norm << "\n";
         }
     }
-    BL_PROFILE_VAR_STOP(blp_up);
 }
 
 // FMG cycle on the coarsest AMR level.
@@ -1461,7 +1476,6 @@ MLMG::computeVolInv ()
 
     if (linop.isCellCentered())
     { 
-        Real temp1, temp2;    
         volinv.resize(namrlevs);
         for (int amrlev = 0; amrlev < namrlevs; ++amrlev) {
             volinv[amrlev].resize(linop.NMGLevels(amrlev));
@@ -1491,6 +1505,7 @@ MLMG::computeVolInv ()
         f(0,mgbottom);
 
 #ifdef AMREX_USE_EB
+        Real temp1, temp2;
         if (rhs[0].hasEBFabFactory())
         {
             ParallelAllReduce::Sum<Real>({volinv[0][0], volinv[0][mgbottom]},
diff --git a/Tools/Backtrace/parse_bt.py b/Tools/Backtrace/parse_bt.py
index c1fb8a5cbd3..cdc64323820 100755
--- a/Tools/Backtrace/parse_bt.py
+++ b/Tools/Backtrace/parse_bt.py
@@ -16,10 +16,25 @@
   lines = f.readlines()
 
 for l in lines:
-  m = re.match("\s*(\d+): \[(0x[\dabcdef]+)\]", l)
+
+  matched = False
+
+  # gnu compiler
+  m = re.match("\s*(\d+): .*\(\+(0x[\dabcdef]+)\)", l)
   if m:
+    matched = True
     frame = m.group(1)
     addr = m.group(2)
+
+  # intel compiler
+  if not matched:
+    m = re.match("\s*(\d+): \[(0x[\dabcdef]+)\]", l)
+    if m:
+      matched = True
+      frame = m.group(1)
+      addr = m.group(2)
+
+  if matched:
     cmd = "addr2line -Cfie %s %s" % (exe_file, addr)
     info = subprocess.check_output(shlex.split(cmd)).decode("utf-8")
     print("%s: %s" % (frame, info))