Repository: pmodels/mpich
Pull request #7367: comm: refactor comm hierarchical information
Author: login: hzhou
Created: 2025-04-03T16:47:41Z | Merged: 2025-04-07T21:42:50Z

## Pull Request Description
`MPIR_Comm` currently contains hierarchical fields including:
```
    MPIR_Comm_hierarchy_kind_t hierarchy_kind;  /* flat, parent, node, or node_roots */
    struct MPIR_Comm *node_comm;        /* Comm of processes in this comm that are on
                                         * the same node as this process. */
    struct MPIR_Comm *node_roots_comm;  /* Comm of root processes for other nodes. */
    int *intranode_table;       /* intranode_table[i] gives the rank in
                                 * node_comm of rank i in this comm or -1 if i
                                 * is not in this process' node_comm.
                                 * It is of size 'local_size'. */
    int *internode_table;       /* internode_table[i] gives the rank in
                                 * node_roots_comm of rank i in this comm.
                                 * It is of size 'local_size'. */
    int node_count;             /* number of nodes this comm is spread over */
```
1. Hierarchical fields are optimizations and they should be optional. A communicator without hierarchical info should still work and be treated as a "flat" topology.
2. The design of these fileds are inefficient, both inflexible and wastes memory. For example, if a communicator is "balanced" and "consecutive" -- a common target for performance engineer -- then it is easier and faster to directly compute rank translation without consulting a table. If the communicator is "irregular", then it is likely not worthwhile for the optimization.
3. `hierarchy_kind` is not particularly useful. All we need is a boolean  attribute to tell whether it communicator has hierarchical structure for optimization or not.
4. The subcomms need be *lightweight* and flexible. They should not be used for querying hierarchical info.

[skip warnings]

## Author Checklist
* [x] **Provide Description** 
      Particularly focus on _why_, not _what_. Reference background, issues, test failures, xfail entries, etc.
* [x] **Commits Follow Good Practice**
      Commits are self-contained and do not do two things at once. 
      Commit message is of the form: `module: short description` 
      Commit message explains what's in the commit.
* [x] **Passes All Tests**
      Whitespace checker. Warnings test. Additional tests via comments.
* [x] **Contribution Agreement**
      For non-Argonne authors, check [contribution agreement](http://www.mpich.org/documentation/contributor-docs/). 
      If necessary, request an explicit comment from your companies PR approval manager.



Diff:
diff --git a/CHANGES b/CHANGES
index 3d1a3bc3dbd..217ad274c7a 100644
--- a/CHANGES
+++ b/CHANGES
@@ -22,6 +22,16 @@
   MPIR_BYTE_INTERNAL, MPIR_CHAR_INTERNAL, MPIR_AINT_INTERNAL instead. There is no
   impact to users.
 
+# Removed MPIR_Find_{local,external}. Added in MPIR_Comm struct with attr and
+  hierarchical fields to more efficiently query communicator's hierarchical
+  structure.
+
+# Use MPIR_Comm_get_{node,node_roots}_comm to obtain node_comm and node_roots_comm.
+  This provides the mechanism to create subcomms on demand. In general, use
+  MPIR_Subcomm_create to create subcomms. Subcomms are marked by
+  MPIR_COMM_ATTR__SUBCOMM bit in the attr field. They can be much more lightweight
+  than user-visible communicators.
+
 # ADI: MPID_Comm_get_lpid removed. Lpids are looked up from the local_group and
   remote_group in the MPIR_Comm struct.
 
diff --git a/src/include/mpir_comm.h b/src/include/mpir_comm.h
index 3c076823daf..3bac35dca59 100644
--- a/src/include/mpir_comm.h
+++ b/src/include/mpir_comm.h
@@ -21,16 +21,6 @@ typedef enum MPIR_Comm_kind_t {
     MPIR_COMM_KIND__INTERCOMM = 1
 } MPIR_Comm_kind_t;
 
-/* ideally we could add these to MPIR_Comm_kind_t, but there's too much existing
- * code that assumes that the only valid values are INTRACOMM or INTERCOMM */
-typedef enum MPIR_Comm_hierarchy_kind_t {
-    MPIR_COMM_HIERARCHY_KIND__FLAT = 0, /* no hierarchy */
-    MPIR_COMM_HIERARCHY_KIND__PARENT = 1,       /* has subcommunicators */
-    MPIR_COMM_HIERARCHY_KIND__NODE_ROOTS = 2,   /* is the subcomm for node roots */
-    MPIR_COMM_HIERARCHY_KIND__NODE = 3, /* is the subcomm for a node */
-    MPIR_COMM_HIERARCHY_KIND__MULTI_LEADS = 4,  /* is the multi_leaders_comm for a node */
-} MPIR_Comm_hierarchy_kind_t;
-
 /* Communicator info hint */
 #define MPIR_COMM_HINT_TYPE_BOOL 0
 #define MPIR_COMM_HINT_TYPE_INT  1
@@ -121,24 +111,24 @@ enum MPIR_COMM_HINT_PREDEFINED_t {
   S*/
 struct MPIR_Comm {
     MPIR_OBJECT_HEADER;         /* adds handle and ref_count fields */
-    MPID_Thread_mutex_t mutex;
+
+    /* -- core fields, required by subcomm as well -- */
+    int attr;                   /* if attr is 0, only the core set of fields are set.
+                                 * Other fields are set in Comm_commit along with corresponding attr bits */
     int context_id;             /* Send context id.  See notes */
     int recvcontext_id;         /* Recv context id (locally allocated).  See notes */
-    int remote_size;            /* Value of MPI_Comm_(remote)_size */
     int rank;                   /* Value of MPI_Comm_rank */
-    MPIR_Attribute *attributes; /* List of attributes */
     int local_size;             /* Value of MPI_Comm_size for local group */
     MPIR_Group *local_group;    /* Groups in communicator. */
-    MPIR_Group *remote_group;   /* The remote group in a inter communicator.
-                                 * Must be NULL in a intra communicator. */
     MPIR_Comm_kind_t comm_kind; /* MPIR_COMM_KIND__INTRACOMM or MPIR_COMM_KIND__INTERCOMM */
-    char name[MPI_MAX_OBJECT_NAME];     /* Required for MPI-2 */
-    MPIR_Errhandler *errhandler;        /* Pointer to the error handler structure */
-    struct MPIR_Comm *local_comm;       /* Defined only for intercomms, holds
-                                         * an intracomm for the local group */
-    struct MPIR_Threadcomm *threadcomm; /* Not NULL only if it's associated with a threadcomm */
+    MPID_Thread_mutex_t mutex;
 
-    MPIR_Comm_hierarchy_kind_t hierarchy_kind;  /* flat, parent, node, or node_roots */
+    /* -- unset unless (attr | MPIR_COMM_ATTR__HIERARCHY) -- */
+    int hierarchy_flags;        /* bit flags for hierarchy charateristics. See bit definitions below. */
+    int local_rank;
+    int num_local;
+    int external_rank;
+    int num_external;
     struct MPIR_Comm *node_comm;        /* Comm of processes in this comm that are on
                                          * the same node as this process. */
     struct MPIR_Comm *node_roots_comm;  /* Comm of root processes for other nodes. */
@@ -149,16 +139,25 @@ struct MPIR_Comm {
     int *internode_table;       /* internode_table[i] gives the rank in
                                  * node_roots_comm of rank i in this comm.
                                  * It is of size 'local_size'. */
-    int node_count;             /* number of nodes this comm is spread over */
 
     int is_low_group;           /* For intercomms only, this boolean is
-                                 * set for all members of one of the
-                                 * two groups of processes and clear for
-                                 * the other.  It enables certain
-                                 * intercommunicator collective operations
-                                 * that wish to use half-duplex operations
-                                 * to implement a full-duplex operation */
+                                 * M* set for all members of one of the
+                                 * * two groups of processes and clear for
+                                 * * the other.  It enables certain
+                                 * * intercommunicator collective operations
+                                 * * that wish to use half-duplex operations
+                                 * * to implement a full-duplex operation */
+    int remote_size;            /* Value of MPI_Comm_(remote)_size */
+    MPIR_Group *remote_group;   /* The remote group in a inter communicator.
+                                 * Must be NULL in a intra communicator. */
+    struct MPIR_Comm *local_comm;       /* Defined only for intercomms, holds
+                                         * an intracomm for the local group */
 
+    /* -- user-level comm required -- */
+    char name[MPI_MAX_OBJECT_NAME];     /* Required for MPI-2 */
+    MPIR_Session *session_ptr;  /* Pointer to MPI session to which the communicator belongs */
+    MPIR_Errhandler *errhandler;        /* Pointer to the error handler structure */
+    MPIR_Attribute *attributes; /* List of attributes */
     struct MPIR_Comm *comm_next;        /* Provides a chain through all active
                                          * communicators */
     struct MPII_Topo_ops *topo_fns;     /* Pointer to a table of functions
@@ -170,6 +169,25 @@ struct MPIR_Comm {
 
     int revoked;                /* Flag to track whether the communicator
                                  * has been revoked */
+
+    /* -- extended features -- */
+    struct MPIR_Threadcomm *threadcomm; /* Not NULL only if it's associated with a threadcomm */
+
+    enum { MPIR_STREAM_COMM_NONE, MPIR_STREAM_COMM_SINGLE, MPIR_STREAM_COMM_MULTIPLEX }
+        stream_comm_type;
+    union {
+        struct {
+            struct MPIR_Stream *stream;
+            int *vci_table;
+        } single;
+        struct {
+            struct MPIR_Stream **local_streams;
+            MPI_Aint *vci_displs;       /* comm size + 1 */
+            int *vci_table;     /* comm size */
+        } multiplex;
+    } stream_comm;
+
+    /* -- optimization fields -- */
     /* A sequence number used for e.g. vci hashing. We can't directly use context_id
      * because context_id is non-sequential and can't be used to identify user-level
      * communicators (due to sub-comms). */
@@ -209,33 +227,29 @@ struct MPIR_Comm {
     } coll;
 
     void *csel_comm;            /* collective selector handle */
+
 #if defined HAVE_HCOLL
     hcoll_comm_priv_t hcoll_priv;
 #endif                          /* HAVE_HCOLL */
 
-    enum { MPIR_STREAM_COMM_NONE, MPIR_STREAM_COMM_SINGLE, MPIR_STREAM_COMM_MULTIPLEX }
-        stream_comm_type;
-    union {
-        struct {
-            struct MPIR_Stream *stream;
-            int *vci_table;
-        } single;
-        struct {
-            struct MPIR_Stream **local_streams;
-            MPI_Aint *vci_displs;       /* comm size + 1 */
-            int *vci_table;     /* comm size */
-        } multiplex;
-    } stream_comm;
-
     MPIR_Request *persistent_requests;
 
     /* Other, device-specific information */
 #ifdef MPID_DEV_COMM_DECL
      MPID_DEV_COMM_DECL
 #endif
-     MPIR_Session * session_ptr;        /* Pointer to MPI session to which the communicator belongs */
 };
 
+/* Bit flags for comm->attr */
+#define MPIR_COMM_ATTR__SUBCOMM   0x1
+#define MPIR_COMM_ATTR__HIERARCHY 0x2
+
+#define MPIR_COMM_HIERARCHY__NO_LOCAL    0x1
+#define MPIR_COMM_HIERARCHY__SINGLE_NODE 0x2
+#define MPIR_COMM_HIERARCHY__NODE_CONSECUTIVE 0x4       /* ranks are ordered by nodes */
+#define MPIR_COMM_HIERARCHY__NODE_BALANCED    0x8       /* same number of ranks on every node */
+#define MPIR_COMM_HIERARCHY__PARENT      0x10   /* has node_comm and node_roots_comm */
+
 #define MPIR_is_self_comm(comm) \
     ((comm)->remote_size == 1 && (comm)->comm_kind == MPIR_COMM_KIND__INTRACOMM && \
      (!(comm)->threadcomm || (comm)->threadcomm->num_threads == 1))
@@ -328,16 +342,72 @@ MPL_STATIC_INLINE_PREFIX int MPIR_Stream_comm_set_attr(MPIR_Comm * comm, int src
     goto fn_exit;
 }
 
+MPL_STATIC_INLINE_PREFIX int MPIR_Get_internode_rank(MPIR_Comm * comm, int r)
+{
+    MPIR_Assert(comm->attr | MPIR_COMM_ATTR__HIERARCHY);
+    if (comm->internode_table) {
+        return comm->internode_table[r];
+    } else {
+        /* canonical or trivial */
+        return r / comm->num_local;
+    }
+}
+
+MPL_STATIC_INLINE_PREFIX int MPIR_Get_intranode_rank(MPIR_Comm * comm, int r)
+{
+    MPIR_Assert(comm->attr | MPIR_COMM_ATTR__HIERARCHY);
+    if (comm->intranode_table) {
+        return comm->intranode_table[r];
+    } else {
+        /* canonical or trivial */
+        if ((comm->rank / comm->num_local) == (r / comm->num_local)) {
+            return r % comm->num_local;
+        } else {
+            return -1;
+        }
+    }
+}
+
+MPL_STATIC_INLINE_PREFIX bool MPII_Comm_is_node_consecutive(MPIR_Comm * comm)
+{
+    return (comm->attr & MPIR_COMM_ATTR__HIERARCHY) &&
+        (comm->hierarchy_flags & MPIR_COMM_HIERARCHY__NODE_CONSECUTIVE);
+}
+
+MPL_STATIC_INLINE_PREFIX bool MPII_Comm_is_node_balanced(MPIR_Comm * comm)
+{
+    return (comm->attr & MPIR_COMM_ATTR__HIERARCHY) &&
+        (comm->hierarchy_flags & MPIR_COMM_HIERARCHY__NODE_BALANCED);
+}
+
+/* node_canonical means node_balanced and node_consecutive */
+MPL_STATIC_INLINE_PREFIX bool MPII_Comm_is_node_canonical(MPIR_Comm * comm)
+{
+    return (comm->attr & MPIR_COMM_ATTR__HIERARCHY) &&
+        (comm->hierarchy_flags & MPIR_COMM_HIERARCHY__NODE_BALANCED) &&
+        (comm->hierarchy_flags & MPIR_COMM_HIERARCHY__NODE_CONSECUTIVE);
+}
+
+MPL_STATIC_INLINE_PREFIX bool MPIR_Comm_is_parent_comm(MPIR_Comm * comm)
+{
+    return (comm->attr & MPIR_COMM_ATTR__HIERARCHY) &&
+        (comm->hierarchy_flags & MPIR_COMM_HIERARCHY__PARENT);
+}
 
 int MPIR_Comm_create(MPIR_Comm **);
 int MPIR_Comm_create_intra(MPIR_Comm * comm_ptr, MPIR_Group * group_ptr, MPIR_Comm ** newcomm_ptr);
 int MPIR_Comm_create_inter(MPIR_Comm * comm_ptr, MPIR_Group * group_ptr, MPIR_Comm ** newcomm_ptr);
 
 
+int MPIR_Subcomm_create(MPIR_Comm * comm, int sub_rank, int sub_size, int *procs,
+                        int context_offset, MPIR_Comm ** subcomm_out);
+int MPIR_Subcomm_free(MPIR_Comm * subcomm);
 int MPIR_Comm_create_subcomms(MPIR_Comm * comm);
 int MPIR_Comm_commit(MPIR_Comm *);
-
-int MPIR_Comm_is_parent_comm(MPIR_Comm *);
+/* we may not always construct comm->node_comm or comm->node_roots_comm. Use
+ * following routines if needed. */
+MPIR_Comm *MPIR_Comm_get_node_comm(MPIR_Comm * comm);
+MPIR_Comm *MPIR_Comm_get_node_roots_comm(MPIR_Comm * comm);
 
 #define MPIR_Comm_rank(comm_ptr) ((comm_ptr)->rank)
 #define MPIR_Comm_size(comm_ptr) ((comm_ptr)->local_size)
@@ -402,9 +472,6 @@ extern struct MPIR_Commops *MPIR_Comm_fns;      /* Communicator creation functio
 
 int MPII_Comm_init(MPIR_Comm *);
 
-int MPII_Comm_is_node_consecutive(MPIR_Comm *);
-int MPII_Comm_is_node_balanced(MPIR_Comm *, int *, bool *);
-
 int MPII_Comm_dup(MPIR_Comm * comm_ptr, MPIR_Info * info, MPIR_Comm ** newcomm_ptr);
 int MPII_Comm_copy(MPIR_Comm * comm_ptr, int size, MPIR_Info * info, MPIR_Comm ** outcomm_ptr);
 int MPII_Comm_copy_data(MPIR_Comm * comm_ptr, MPIR_Info * info, MPIR_Comm ** outcomm_ptr);
diff --git a/src/include/mpir_misc.h b/src/include/mpir_misc.h
index 50c490357f0..60cfa275376 100644
--- a/src/include/mpir_misc.h
+++ b/src/include/mpir_misc.h
@@ -113,13 +113,6 @@ int MPIR_Ilocalcopy_gpu(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype se
 @*/
 void MPIR_Add_finalize(int (*routine) (void *), void *extra, int priority);
 
-/* Routines for determining local and remote processes */
-int MPIR_Find_local(struct MPIR_Comm *comm, int *local_size_p, int *local_rank_p,
-                    int **local_ranks_p, int **intranode_table);
-int MPIR_Find_external(struct MPIR_Comm *comm, int *external_size_p, int *external_rank_p,
-                       int **external_ranks_p, int **internode_table_p);
-int MPIR_Get_internode_rank(MPIR_Comm * comm_ptr, int r);
-int MPIR_Get_intranode_rank(MPIR_Comm * comm_ptr, int r);
 
 #define MPIR_CAST(T, val) CAST_##T((val))
 #ifdef NDEBUG
diff --git a/src/mpi/coll/mpir_coll_sched_auto.c b/src/mpi/coll/mpir_coll_sched_auto.c
index 8b118bda349..97791f32b1b 100644
--- a/src/mpi/coll/mpir_coll_sched_auto.c
+++ b/src/mpi/coll/mpir_coll_sched_auto.c
@@ -41,7 +41,7 @@ int MPIR_Ibcast_intra_sched_auto(void *buffer, MPI_Aint count, MPI_Datatype data
 
     MPIR_Assert(comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM);
 
-    if (comm_ptr->hierarchy_kind == MPIR_COMM_HIERARCHY_KIND__PARENT) {
+    if (MPIR_Comm_is_parent_comm(comm_ptr)) {
         mpi_errno = MPIR_Ibcast_intra_sched_smp(buffer, count, datatype, root, comm_ptr, s);
         if (mpi_errno)
             MPIR_ERR_POP(mpi_errno);
@@ -526,7 +526,7 @@ int MPIR_Ireduce_intra_sched_auto(const void *sendbuf, void *recvbuf, MPI_Aint c
 
     MPIR_Assert(comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM);
 
-    if (comm_ptr->hierarchy_kind == MPIR_COMM_HIERARCHY_KIND__PARENT && MPIR_Op_is_commutative(op)) {
+    if (MPIR_Comm_is_parent_comm(comm_ptr) && MPIR_Op_is_commutative(op)) {
         mpi_errno = MPIR_Ireduce_intra_sched_smp(sendbuf, recvbuf, count,
                                                  datatype, op, root, comm_ptr, s);
         if (mpi_errno)
@@ -582,7 +582,7 @@ int MPIR_Iallreduce_intra_sched_auto(const void *sendbuf, void *recvbuf, MPI_Ain
 
     MPIR_Assert(comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM);
 
-    if (comm_ptr->hierarchy_kind == MPIR_COMM_HIERARCHY_KIND__PARENT && MPIR_Op_is_commutative(op)) {
+    if (MPIR_Comm_is_parent_comm(comm_ptr) && MPIR_Op_is_commutative(op)) {
         mpi_errno =
             MPIR_Iallreduce_intra_sched_smp(sendbuf, recvbuf, count, datatype, op, comm_ptr, s);
         if (mpi_errno)
@@ -792,7 +792,7 @@ int MPIR_Iscan_intra_sched_auto(const void *sendbuf, void *recvbuf, MPI_Aint cou
 {
     int mpi_errno = MPI_SUCCESS;
 
-    if (comm_ptr->hierarchy_kind == MPIR_COMM_HIERARCHY_KIND__PARENT) {
+    if (MPIR_Comm_is_parent_comm(comm_ptr)) {
         mpi_errno = MPIR_Iscan_intra_sched_smp(sendbuf, recvbuf, count, datatype, op, comm_ptr, s);
     } else {
         mpi_errno =
diff --git a/src/mpi/coll/src/csel.c b/src/mpi/coll/src/csel.c
index 1253f847f66..c07cf91a709 100644
--- a/src/mpi/coll/src/csel.c
+++ b/src/mpi/coll/src/csel.c
@@ -114,7 +114,7 @@ typedef struct csel_node {
             int val;
         } comm_avg_ppn_lt;
         struct {
-            MPIR_Comm_hierarchy_kind_t val;
+            bool val;
         } comm_hierarchy;
         struct {
             void *container;
@@ -213,16 +213,10 @@ static void print_tree(csel_node_s * node)
             nprintf("all blocks have the same count\n");
             break;
         case CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY:
-            if (node->u.comm_hierarchy.val == MPIR_COMM_HIERARCHY_KIND__PARENT)
-                nprintf("communicator is the parent comm\n");
-            else if (node->u.comm_hierarchy.val == MPIR_COMM_HIERARCHY_KIND__NODE_ROOTS)
-                nprintf("communicator is the node_roots comm\n");
-            else if (node->u.comm_hierarchy.val == MPIR_COMM_HIERARCHY_KIND__NODE)
-                nprintf("communicator is the node comm\n");
-            else if (node->u.comm_hierarchy.val == MPIR_COMM_HIERARCHY_KIND__FLAT)
-                nprintf("communicator is a flat comm\n");
+            if (node->u.comm_hierarchy.val)
+                nprintf("communicator has hierarchical structure\n");
             else
-                nprintf("communicator is of any hierarchy kind\n");
+                nprintf("communicator does not have hierarchical structure\n");
             break;
         case CSEL_NODE_TYPE__OPERATOR__IS_NODE_CONSECUTIVE:
             nprintf("process ranks are consecutive on the node\n");
@@ -526,16 +520,16 @@ static csel_node_s *parse_json_tree(struct json_object *obj,
             tmp->u.comm_avg_ppn_le.val = atoi(ckey + strlen("comm_avg_ppn<"));
         } else if (!strcmp(ckey, "comm_hierarchy=parent")) {
             tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY;
-            tmp->u.comm_hierarchy.val = MPIR_COMM_HIERARCHY_KIND__PARENT;
+            tmp->u.comm_hierarchy.val = true;
         } else if (!strcmp(ckey, "comm_hierarchy=node_roots")) {
             tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY;
-            tmp->u.comm_hierarchy.val = MPIR_COMM_HIERARCHY_KIND__NODE_ROOTS;
+            tmp->u.comm_hierarchy.val = false;
         } else if (!strcmp(ckey, "comm_hierarchy=node")) {
             tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY;
-            tmp->u.comm_hierarchy.val = MPIR_COMM_HIERARCHY_KIND__NODE;
+            tmp->u.comm_hierarchy.val = false;
         } else if (!strcmp(ckey, "comm_hierarchy=flat")) {
             tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY;
-            tmp->u.comm_hierarchy.val = MPIR_COMM_HIERARCHY_KIND__FLAT;
+            tmp->u.comm_hierarchy.val = false;
         } else if (key_is_any(ckey)) {
             tmp->type = CSEL_NODE_TYPE__OPERATOR__ANY;
         } else {
@@ -630,8 +624,7 @@ static csel_node_s *prune_tree(csel_node_s * root, MPIR_Comm * comm_ptr)
                 break;
 
             case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_NODE_COMM_SIZE:
-                if (comm_ptr->node_comm != NULL &&
-                    MPIR_Comm_size(comm_ptr) == MPIR_Comm_size(comm_ptr->node_comm))
+                if ((comm_ptr->attr & MPIR_COMM_ATTR__HIERARCHY) && comm_ptr->num_external == 1)
                     node = node->success;
                 else
                     node = node->failure;
@@ -645,7 +638,7 @@ static csel_node_s *prune_tree(csel_node_s * root, MPIR_Comm * comm_ptr)
                 break;
 
             case CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY:
-                if (comm_ptr->hierarchy_kind == node->u.comm_hierarchy.val)
+                if (MPIR_Comm_is_parent_comm(comm_ptr) == node->u.comm_hierarchy.val)
                     node = node->success;
                 else
                     node = node->failure;
@@ -659,14 +652,16 @@ static csel_node_s *prune_tree(csel_node_s * root, MPIR_Comm * comm_ptr)
                 break;
 
             case CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LE:
-                if (comm_ptr->local_size <= node->u.comm_avg_ppn_le.val * comm_ptr->node_count)
+                if ((comm_ptr->attr & MPIR_COMM_ATTR__HIERARCHY) &&
+                    comm_ptr->local_size <= node->u.comm_avg_ppn_le.val * comm_ptr->num_external)
                     node = node->success;
                 else
                     node = node->failure;
                 break;
 
             case CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LT:
-                if (comm_ptr->local_size < node->u.comm_avg_ppn_le.val * comm_ptr->node_count)
+                if ((comm_ptr->attr & MPIR_COMM_ATTR__HIERARCHY) &&
+                    comm_ptr->local_size < node->u.comm_avg_ppn_le.val * comm_ptr->num_external)
                     node = node->success;
                 else
                     node = node->failure;
@@ -1229,8 +1224,7 @@ void *MPIR_Csel_search(void *csel_, MPIR_Csel_coll_sig_s coll_info)
                 break;
 
             case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_NODE_COMM_SIZE:
-                if (comm_ptr->node_comm != NULL &&
-                    MPIR_Comm_size(comm_ptr) == MPIR_Comm_size(comm_ptr->node_comm))
+                if ((comm_ptr->attr & MPIR_COMM_ATTR__HIERARCHY) && comm_ptr->num_external == 1)
                     node = node->success;
                 else
                     node = node->failure;
@@ -1329,21 +1323,23 @@ void *MPIR_Csel_search(void *csel_, MPIR_Csel_coll_sig_s coll_info)
                 break;
 
             case CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LE:
-                if (comm_ptr->local_size <= node->u.comm_avg_ppn_le.val * comm_ptr->node_count)
+                if ((comm_ptr->attr & MPIR_COMM_ATTR__HIERARCHY) &&
+                    comm_ptr->local_size <= node->u.comm_avg_ppn_le.val * comm_ptr->num_external)
                     node = node->success;
                 else
                     node = node->failure;
                 break;
 
             case CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LT:
-                if (comm_ptr->local_size < node->u.comm_avg_ppn_le.val * comm_ptr->node_count)
+                if ((comm_ptr->attr & MPIR_COMM_ATTR__HIERARCHY) &&
+                    comm_ptr->local_size < node->u.comm_avg_ppn_le.val * comm_ptr->num_external)
                     node = node->success;
                 else
                     node = node->failure;
                 break;
 
             case CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY:
-                if (coll_info.comm_ptr->hierarchy_kind == node->u.comm_hierarchy.val)
+                if (MPIR_Comm_is_parent_comm(comm_ptr) == node->u.comm_hierarchy.val)
                     node = node->success;
                 else
                     node = node->failure;
diff --git a/src/mpi/comm/commutil.c b/src/mpi/comm/commutil.c
index 0a5168249ae..c5e3dc25255 100644
--- a/src/mpi/comm/commutil.c
+++ b/src/mpi/comm/commutil.c
@@ -27,7 +27,6 @@ MPIR_Object_alloc_t MPIR_Comm_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_COMM,
 
 /* Communicator creation functions */
 struct MPIR_Commops *MPIR_Comm_fns = NULL;
-static int MPIR_Comm_commit_internal(MPIR_Comm * comm);
 
 /* Communicator hint functions */
 /* For balance of simplicity and feature, we'll internally use integers for both keys
@@ -274,6 +273,9 @@ int MPII_Comm_init(MPIR_Comm * comm_p)
 
     MPIR_Object_set_ref(comm_p, 1);
 
+    comm_p->attr = 0;
+    comm_p->hierarchy_flags = 0;
+
     /* initialize local and remote sizes to -1 to allow other parts of
      * the stack to detect errors more easily */
     comm_p->local_size = -1;
@@ -297,7 +299,6 @@ int MPII_Comm_init(MPIR_Comm * comm_p)
         }
     }
 
-    comm_p->hierarchy_kind = MPIR_COMM_HIERARCHY_KIND__FLAT;
     comm_p->node_comm = NULL;
     comm_p->node_roots_comm = NULL;
     comm_p->intranode_table = NULL;
@@ -419,70 +420,205 @@ int MPII_Setup_intercomm_localcomm(MPIR_Comm * intercomm_ptr)
     return mpi_errno;
 }
 
-static int get_node_count(MPIR_Comm * comm, int *node_count)
+/* Copy relevant hints and settings to a given subcomm */
+static void propagate_subcomms(MPIR_Comm * comm, MPIR_Comm * subcomm)
 {
-    int mpi_errno = MPI_SUCCESS;
-    struct uniq_nodes {
-        int id;
-        UT_hash_handle hh;
-    } *node_list = NULL;
-    struct uniq_nodes *s, *tmp;
-
-    if (comm->comm_kind != MPIR_COMM_KIND__INTRACOMM) {
-        *node_count = comm->local_size;
-        goto fn_exit;
-    } else if (comm->hierarchy_kind == MPIR_COMM_HIERARCHY_KIND__NODE) {
-        *node_count = 1;
-        goto fn_exit;
-    } else if (comm->hierarchy_kind == MPIR_COMM_HIERARCHY_KIND__NODE_ROOTS) {
-        *node_count = comm->local_size;
-        goto fn_exit;
+    /* Copy vci hints */
+    subcomm->hints[MPIR_COMM_HINT_SENDER_VCI] = comm->hints[MPIR_COMM_HINT_SENDER_VCI];
+    subcomm->hints[MPIR_COMM_HINT_RECEIVER_VCI] = comm->hints[MPIR_COMM_HINT_RECEIVER_VCI];
+    subcomm->hints[MPIR_COMM_HINT_VCI] = comm->hints[MPIR_COMM_HINT_VCI];
+
+    subcomm->vcis_enabled = comm->vcis_enabled;
+    subcomm->seq = comm->seq;
+}
+
+static int get_max_num_nodes(void)
+{
+    if (MPIR_Process.node_hostnames) {
+        return utarray_len(MPIR_Process.node_hostnames);
+    } else {
+        return MPIR_Process.num_nodes;
     }
+}
+
+static int check_hierarchy(MPIR_Comm * comm)
+{
+    int mpi_errno = MPI_SUCCESS;
+    MPIR_CHKLMEM_DECL();
+    MPIR_CHKPMEM_DECL();
+
+    int comm_size = comm->local_size;
+
+    /* internode_table[rank] stores the (relabled) node_id (0..num_nodes-1) */
+    int *internode_table;
+    MPIR_CHKPMEM_MALLOC(internode_table, comm_size * sizeof(int), MPL_MEM_COMM);
 
-    /* go through the list of ranks and add the unique ones to the
-     * node_list array */
-    for (int i = 0; i < comm->local_size; i++) {
-        int node;
+    /* nodes[node_id] relabels node_id to 0..(num_nodes-1) */
+    int *nodes;
+    int max_nodes = get_max_num_nodes();
+    MPIR_CHKLMEM_MALLOC(nodes, max_nodes * sizeof(int));
+    for (int i = 0; i < max_nodes; i++) {
+        nodes[i] = -1;
+    }
 
-        mpi_errno = MPID_Get_node_id(comm, i, &node);
+    /* populate internode_table */
+    int num_nodes = 0;
+    int my_node_id = -1;
+    for (int i = 0; i < comm_size; i++) {
+        int node_id;
+        mpi_errno = MPID_Get_node_id(comm, i, &node_id);
         MPIR_ERR_CHECK(mpi_errno);
 
-        HASH_FIND_INT(node_list, &node, s);
-        if (s == NULL) {
-            s = (struct uniq_nodes *) MPL_malloc(sizeof(struct uniq_nodes), MPL_MEM_COLL);
-            MPIR_Assert(s);
-            s->id = node;
-            HASH_ADD_INT(node_list, id, s, MPL_MEM_COLL);
+        if (node_id == -1) {
+            /* Hierarchy not supported */
+            goto fn_fail;
+        }
+
+        if (nodes[node_id] == -1) {
+            nodes[node_id] = num_nodes++;
+        }
+        internode_table[i] = nodes[node_id];
+        if (i == comm->rank) {
+            my_node_id = nodes[node_id];
+        }
+    }
+
+    /* intranode_table[rank] stores the local rank (or -1 if not on the same node) */
+    int *intranode_table;
+    MPIR_CHKPMEM_MALLOC(intranode_table, comm_size * sizeof(int), MPL_MEM_COMM);
+
+    /* get local node */
+    int num_local = 0;
+    int local_rank = 0;
+    for (int i = 0; i < comm_size; i++) {
+        if (internode_table[i] == my_node_id) {
+            intranode_table[i] = num_local;
+            if (i == comm->rank) {
+                local_rank = num_local;
+            }
+            num_local++;
+        } else {
+            intranode_table[i] = -1;
         }
     }
 
-    /* the final size of our hash table is our node count */
-    *node_count = HASH_COUNT(node_list);
+    comm->attr |= MPIR_COMM_ATTR__HIERARCHY;
+
+    comm->local_rank = local_rank;
+    comm->num_local = num_local;
+    comm->num_external = num_nodes;
+    comm->external_rank = my_node_id;
+    comm->internode_table = internode_table;
+    comm->intranode_table = intranode_table;
+    comm->node_comm = NULL;
+    comm->node_roots_comm = NULL;
+
+    comm->hierarchy_flags = 0;
+    bool skip_tables = false;
+    if (num_nodes == 1) {
+        comm->hierarchy_flags |= MPIR_COMM_HIERARCHY__SINGLE_NODE;
+        skip_tables = true;
+    } else if (num_nodes == comm_size) {
+        comm->hierarchy_flags |= MPIR_COMM_HIERARCHY__NO_LOCAL;
+        skip_tables = true;
+    } else {
+        /* check whether all ranks are ordered by node_id */
+        bool is_node_consecutive = true;
+        for (int i = 1; i < comm_size; i++) {
+            if (internode_table[i] < internode_table[i - 1]) {
+                is_node_consecutive = false;
+                break;
+            }
+        }
 
-    /* free up everything */
-    HASH_ITER(hh, node_list, s, tmp) {
-        HASH_DEL(node_list, s);
-        MPL_free(s);
+        /* check whether every node has the same number of ranks */
+        bool is_node_balanced = true;
+        int *node_counts;
+        MPIR_CHKLMEM_MALLOC(node_counts, num_nodes * sizeof(int));
+        for (int i = 0; i < num_nodes; i++) {
+            node_counts[i] = 0;
+        }
+        for (int i = 0; i < comm_size; i++) {
+            node_counts[internode_table[i]]++;
+        }
+        for (int i = 0; i < num_nodes; i++) {
+            if (node_counts[i] != num_local) {
+                is_node_balanced = false;
+                break;
+            }
+        }
+
+        /* set hierarchy_flags */
+        if (is_node_consecutive) {
+            comm->hierarchy_flags |= MPIR_COMM_HIERARCHY__NODE_CONSECUTIVE;
+        }
+        if (is_node_balanced) {
+            comm->hierarchy_flags |= MPIR_COMM_HIERARCHY__NODE_BALANCED;
+        }
+        if (is_node_consecutive && is_node_balanced) {
+            skip_tables = true;
+        }
+    }
+
+    if (skip_tables) {
+        comm->internode_table = NULL;
+        comm->intranode_table = NULL;
+        goto fn_reap;
     }
 
   fn_exit:
+    MPIR_CHKLMEM_FREEALL();
     return mpi_errno;
   fn_fail:
+  fn_reap:
+    MPIR_CHKPMEM_REAP();
     goto fn_exit;
 }
 
-static int MPIR_Comm_commit_internal(MPIR_Comm * comm)
+/* subcomms are internal comms
+ * * always intracomm
+ * * never have hierarchical info (i.e. flat)
+ */
+int MPIR_Subcomm_create(MPIR_Comm * comm, int sub_size, int sub_rank, int *procs,
+                        int context_offset, MPIR_Comm ** subcomm_out)
 {
     int mpi_errno = MPI_SUCCESS;
+    MPIR_Comm *subcomm;
     MPIR_FUNC_ENTER;
 
+    mpi_errno = MPIR_Comm_create(&subcomm);
+    MPIR_ERR_CHECK(mpi_errno);
+
+    subcomm->attr |= MPIR_COMM_ATTR__SUBCOMM;
+
+    subcomm->context_id = comm->context_id + context_offset;
+    subcomm->recvcontext_id = subcomm->context_id;
+    subcomm->comm_kind = MPIR_COMM_KIND__INTRACOMM;
+
+    subcomm->rank = sub_rank;
+    subcomm->local_size = sub_size;
+    subcomm->remote_size = sub_size;
+
+    /* construct local_group */
+    MPIR_Group *parent_group = comm->local_group;
+    MPIR_Assert(parent_group);
+    mpi_errno = MPIR_Group_incl_impl(parent_group, sub_size, procs, &subcomm->local_group);
+    MPIR_ERR_CHECK(mpi_errno);
+
     /* Notify device of communicator creation */
-    mpi_errno = MPID_Comm_commit_pre_hook(comm);
+    mpi_errno = MPID_Comm_commit_pre_hook(subcomm);
     MPIR_ERR_CHECK(mpi_errno);
 
-    mpi_errno = get_node_count(comm, &comm->node_count);
+    /* Create collectives-specific infrastructure */
+    mpi_errno = MPIR_Coll_comm_init(subcomm);
+    MPIR_ERR_CHECK(mpi_errno);
+
+    /* call post commit hooks */
+    mpi_errno = MPID_Comm_commit_post_hook(subcomm);
     MPIR_ERR_CHECK(mpi_errno);
 
+    *subcomm_out = subcomm;
+
   fn_exit:
     MPIR_FUNC_EXIT;
     return mpi_errno;
@@ -490,159 +626,135 @@ static int MPIR_Comm_commit_internal(MPIR_Comm * comm)
     goto fn_exit;
 }
 
-/* Copy relevant hints to a given subcomm */
-static void propagate_hints_to_subcomm(MPIR_Comm * comm, MPIR_Comm * subcomm)
-{
-    /* Copy vci hints */
-    subcomm->hints[MPIR_COMM_HINT_SENDER_VCI] = comm->hints[MPIR_COMM_HINT_SENDER_VCI];
-    subcomm->hints[MPIR_COMM_HINT_RECEIVER_VCI] = comm->hints[MPIR_COMM_HINT_RECEIVER_VCI];
-    subcomm->hints[MPIR_COMM_HINT_VCI] = comm->hints[MPIR_COMM_HINT_VCI];
-}
-
-static void propagate_vcis_enabled(MPIR_Comm * comm)
-{
-    if (comm->node_comm != NULL)
-        comm->node_comm->vcis_enabled = comm->vcis_enabled;
-
-    if (comm->node_roots_comm != NULL)
-        comm->node_roots_comm->vcis_enabled = comm->vcis_enabled;
-}
-
-int MPIR_Comm_create_subcomms(MPIR_Comm * comm)
+int MPIR_Subcomm_free(MPIR_Comm * subcomm)
 {
     int mpi_errno = MPI_SUCCESS;
-    int num_local = -1, num_external = -1;
-    int local_rank = -1, external_rank = -1;
-    int *local_procs = NULL, *external_procs = NULL;
-
     MPIR_FUNC_ENTER;
 
-    MPIR_Assert(comm->node_comm == NULL);
-    MPIR_Assert(comm->node_roots_comm == NULL);
-
-    mpi_errno = MPIR_Find_local(comm, &num_local, &local_rank, &local_procs,
-                                &comm->intranode_table);
-    /* --BEGIN ERROR HANDLING-- */
-    if (mpi_errno) {
-        if (MPIR_Err_is_fatal(mpi_errno))
-            MPIR_ERR_POP(mpi_errno);
-
-        /* Non-fatal errors simply mean that this communicator will not have
-         * any node awareness.  Node-aware collectives are an optimization. */
-        MPL_DBG_MSG_P(MPIR_DBG_COMM, VERBOSE, "MPIR_Find_local failed for comm_ptr=%p", comm);
-        MPL_free(comm->intranode_table);
-
-        mpi_errno = MPI_SUCCESS;
-        goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
-
-    mpi_errno = MPIR_Find_external(comm, &num_external, &external_rank, &external_procs,
-                                   &comm->internode_table);
-    /* --BEGIN ERROR HANDLING-- */
-    if (mpi_errno) {
-        if (MPIR_Err_is_fatal(mpi_errno))
-            MPIR_ERR_POP(mpi_errno);
-
-        /* Non-fatal errors simply mean that this communicator will not have
-         * any node awareness.  Node-aware collectives are an optimization. */
-        MPL_DBG_MSG_P(MPIR_DBG_COMM, VERBOSE, "MPIR_Find_external failed for comm_ptr=%p", comm);
-        MPL_free(comm->internode_table);
-
-        mpi_errno = MPI_SUCCESS;
-        goto fn_exit;
-    }
-    /* --END ERROR HANDLING-- */
+    mpi_errno = MPII_Coll_comm_cleanup(subcomm);
+    MPIR_ERR_CHECK(mpi_errno);
 
-    /* defensive checks */
-    MPIR_Assert(num_local > 0);
-    MPIR_Assert(num_local > 1 || external_rank >= 0);
-    MPIR_Assert(external_rank < 0 || external_procs != NULL);
+    mpi_errno = MPID_Comm_free_hook(subcomm);
+    MPIR_ERR_CHECK(mpi_errno);
 
-    /* if the node_roots_comm and comm would be the same size, then creating
-     * the second communicator is useless and wasteful. */
-    if (num_external == comm->remote_size) {
-        MPIR_Assert(num_local == 1);
-        goto fn_exit;
-    }
+    mpi_errno = MPIR_Group_free_impl(subcomm->local_group);
+    MPIR_ERR_CHECK(mpi_errno);
 
-    /* we don't need a local comm if this process is the only one on this node */
-    if (num_local > 1) {
-        mpi_errno = MPIR_Comm_create(&comm->node_comm);
-        MPIR_ERR_CHECK(mpi_errno);
+    MPIR_Handle_obj_free(&MPIR_Comm_mem, subcomm);
 
-        comm->node_comm->context_id = comm->context_id + MPIR_CONTEXT_INTRANODE_OFFSET;
-        comm->node_comm->recvcontext_id = comm->node_comm->context_id;
-        comm->node_comm->rank = local_rank;
-        comm->node_comm->comm_kind = MPIR_COMM_KIND__INTRACOMM;
-        comm->node_comm->hierarchy_kind = MPIR_COMM_HIERARCHY_KIND__NODE;
-        comm->node_comm->local_comm = NULL;
-        MPL_DBG_MSG_D(MPIR_DBG_COMM, VERBOSE, "Create node_comm=%p\n", comm->node_comm);
+  fn_exit:
+    MPIR_FUNC_EXIT;
+    return mpi_errno;
+  fn_fail:
+    goto fn_exit;
+}
 
-        comm->node_comm->local_size = num_local;
-        comm->node_comm->remote_size = num_local;
+int MPIR_Comm_create_subcomms(MPIR_Comm * comm)
+{
+    int mpi_errno = MPI_SUCCESS;
+    MPIR_FUNC_ENTER;
+    MPIR_CHKLMEM_DECL();
 
-        MPIR_Comm_set_session_ptr(comm->node_comm, comm->session_ptr);
+    int comm_size = comm->local_size;
 
-        /* Copy relevant hints to node_comm */
-        propagate_hints_to_subcomm(comm, comm->node_comm);
+    /* -- node_comm -- */
+    if (comm->num_local > 1 && comm->node_comm == NULL) {
+        int *local_procs;
+        MPIR_CHKLMEM_MALLOC(local_procs, comm->num_local * sizeof(int));
 
-        /* construct local_group */
-        MPIR_Group *parent_group = comm->local_group;
-        MPIR_Assert(parent_group);
-        mpi_errno = MPIR_Group_incl_impl(parent_group, num_local, local_procs,
-                                         &comm->node_comm->local_group);
-        MPIR_ERR_CHECK(mpi_errno);
+        if (comm->internode_table) {
+            int r = 0;
+            for (int i = 0; i < comm_size; i++) {
+                if (comm->internode_table[i] == comm->external_rank) {
+                    local_procs[r++] = i;
+                }
+            }
+        } else {
+            /* canonical or trivial */
+            for (int i = 0; i < comm->num_local; i++) {
+                local_procs[i] = comm->external_rank * comm->num_local + i;
+            }
+        }
 
-        mpi_errno = MPIR_Comm_commit_internal(comm->node_comm);
+        mpi_errno = MPIR_Subcomm_create(comm, comm->num_local, comm->local_rank,
+                                        local_procs, MPIR_CONTEXT_INTRANODE_OFFSET,
+                                        &comm->node_comm);
         MPIR_ERR_CHECK(mpi_errno);
     }
 
-    /* this process may not be a member of the node_roots_comm */
-    if (local_rank == 0) {
-        mpi_errno = MPIR_Comm_create(&comm->node_roots_comm);
-        MPIR_ERR_CHECK(mpi_errno);
-
-        comm->node_roots_comm->context_id = comm->context_id + MPIR_CONTEXT_INTERNODE_OFFSET;
-        comm->node_roots_comm->recvcontext_id = comm->node_roots_comm->context_id;
-        comm->node_roots_comm->rank = external_rank;
-        comm->node_roots_comm->comm_kind = MPIR_COMM_KIND__INTRACOMM;
-        comm->node_roots_comm->hierarchy_kind = MPIR_COMM_HIERARCHY_KIND__NODE_ROOTS;
-        comm->node_roots_comm->local_comm = NULL;
-        MPL_DBG_MSG_D(MPIR_DBG_COMM, VERBOSE, "Create node_roots_comm=%p\n", comm->node_roots_comm);
-
-        comm->node_roots_comm->local_size = num_external;
-        comm->node_roots_comm->remote_size = num_external;
-
-        MPIR_Comm_set_session_ptr(comm->node_roots_comm, comm->session_ptr);
-
-        /* Copy relevant hints to node_roots_comm */
-        propagate_hints_to_subcomm(comm, comm->node_roots_comm);
-
-        /* construct local_group */
-        MPIR_Group *parent_group = comm->local_group;
-        MPIR_Assert(parent_group);
-        mpi_errno = MPIR_Group_incl_impl(parent_group, num_external, external_procs,
-                                         &comm->node_roots_comm->local_group);
-        MPIR_ERR_CHECK(mpi_errno);
+    /* -- node_roots_comm -- */
+    if (comm->local_rank == 0 && comm->node_roots_comm == NULL) {
+        int *external_procs;
+        MPIR_CHKLMEM_MALLOC(external_procs, comm->num_external * sizeof(int));
+
+        if (comm->internode_table) {
+            int r = 0;
+            for (int i = 0; i < comm_size; i++) {
+                if (comm->internode_table[i] >= r) {
+                    MPIR_Assert(comm->internode_table[i] == r);
+                    external_procs[r++] = i;
+                }
+            }
+        } else {
+            /* canonical or trivial */
+            for (int i = 0; i < comm->num_external; i++) {
+                external_procs[i] = i * comm->num_local;
+            }
+        }
 
-        mpi_errno = MPIR_Comm_commit_internal(comm->node_roots_comm);
+        mpi_errno = MPIR_Subcomm_create(comm, comm->num_external, comm->external_rank,
+                                        external_procs, MPIR_CONTEXT_INTERNODE_OFFSET,
+                                        &comm->node_roots_comm);
         MPIR_ERR_CHECK(mpi_errno);
     }
 
-    propagate_vcis_enabled(comm);
-
-    comm->hierarchy_kind = MPIR_COMM_HIERARCHY_KIND__PARENT;
+    comm->hierarchy_flags |= MPIR_COMM_HIERARCHY__PARENT;
 
   fn_exit:
-    MPL_free(local_procs);
-    MPL_free(external_procs);
+    MPIR_CHKLMEM_FREEALL();
     MPIR_FUNC_EXIT;
     return mpi_errno;
   fn_fail:
     goto fn_exit;
 }
 
+MPIR_Comm *MPIR_Comm_get_node_comm(MPIR_Comm * comm)
+{
+    MPIR_Assert(comm->attr & MPIR_COMM_ATTR__HIERARCHY);
+    if (comm->num_external == 1) {
+        return comm;
+    } else if (comm->num_local == 1) {
+        if (!MPIR_Process.comm_self) {
+            int mpi_errno = MPIR_init_comm_self();
+            MPIR_Assertp(mpi_errno == MPI_SUCCESS);
+        }
+        return MPIR_Process.comm_self;
+    } else {
+        /* TODO: construct node_comm on-demand */
+        MPIR_Assert(comm->node_comm);
+        return comm->node_comm;
+    }
+}
+
+MPIR_Comm *MPIR_Comm_get_node_roots_comm(MPIR_Comm * comm)
+{
+    MPIR_Assert(comm->attr & MPIR_COMM_ATTR__HIERARCHY);
+    MPIR_Assert(comm->local_rank == 0);
+    if (comm->num_external == comm->local_size) {
+        return comm;
+    } else if (comm->num_external == 1) {
+        if (!MPIR_Process.comm_self) {
+            int mpi_errno = MPIR_init_comm_self();
+            MPIR_Assertp(mpi_errno == MPI_SUCCESS);
+        }
+        return MPIR_Process.comm_self;
+    } else {
+        /* TODO: construct node_roots_comm on-demand */
+        MPIR_Assert(comm->node_roots_comm);
+        return comm->node_roots_comm;
+    }
+}
+
 /* static routines for MPIR_Comm_commit */
 static int init_comm_seq(MPIR_Comm * comm)
 {
@@ -668,14 +780,6 @@ static int init_comm_seq(MPIR_Comm * comm)
         comm->seq = tmp;
     }
 
-    if (comm->node_comm) {
-        comm->node_comm->seq = comm->seq;
-    }
-
-    if (comm->node_roots_comm) {
-        comm->node_roots_comm->seq = comm->seq;
-    }
-
   fn_exit:
     return mpi_errno;
   fn_fail:
@@ -690,35 +794,21 @@ static int init_comm_seq(MPIR_Comm * comm)
 int MPIR_Comm_commit(MPIR_Comm * comm)
 {
     int mpi_errno = MPI_SUCCESS;
-
     MPIR_FUNC_ENTER;
 
-    /* It's OK to relax these assertions, but we should do so very
-     * intentionally.  For now this function is the only place that we create
-     * our hierarchy of communicators */
-    MPIR_Assert(comm->node_comm == NULL);
-    MPIR_Assert(comm->node_roots_comm == NULL);
+    mpi_errno = check_hierarchy(comm);
+    MPIR_ERR_CHECK(mpi_errno);
 
     /* Notify device of communicator creation */
-    mpi_errno = MPIR_Comm_commit_internal(comm);
+    mpi_errno = MPID_Comm_commit_pre_hook(comm);
     MPIR_ERR_CHECK(mpi_errno);
 
-    if (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM && !MPIR_CONTEXT_READ_FIELD(SUBCOMM, comm->context_id)) {  /*make sure this is not a subcomm */
-        mpi_errno = MPIR_Comm_create_subcomms(comm);
-        MPIR_ERR_CHECK(mpi_errno);
-    }
-
     /* Create collectives-specific infrastructure */
     mpi_errno = MPIR_Coll_comm_init(comm);
     MPIR_ERR_CHECK(mpi_errno);
 
-    if (comm->node_comm) {
-        mpi_errno = MPIR_Coll_comm_init(comm->node_comm);
-        MPIR_ERR_CHECK(mpi_errno);
-    }
-
-    if (comm->node_roots_comm) {
-        mpi_errno = MPIR_Coll_comm_init(comm->node_roots_comm);
+    if ((comm->attr & MPIR_COMM_ATTR__HIERARCHY) && (comm->num_external != comm->local_size)) {
+        mpi_errno = MPIR_Comm_create_subcomms(comm);
         MPIR_ERR_CHECK(mpi_errno);
     }
 
@@ -726,19 +816,16 @@ int MPIR_Comm_commit(MPIR_Comm * comm)
     mpi_errno = MPID_Comm_commit_post_hook(comm);
     MPIR_ERR_CHECK(mpi_errno);
 
-    if (comm->node_comm) {
-        mpi_errno = MPID_Comm_commit_post_hook(comm->node_comm);
-        MPIR_ERR_CHECK(mpi_errno);
-    }
-
-    if (comm->node_roots_comm) {
-        mpi_errno = MPID_Comm_commit_post_hook(comm->node_roots_comm);
-        MPIR_ERR_CHECK(mpi_errno);
-    }
-
     if (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM && comm->vcis_enabled) {
         mpi_errno = init_comm_seq(comm);
         MPIR_ERR_CHECK(mpi_errno);
+
+        if (comm->node_comm) {
+            propagate_subcomms(comm, comm->node_comm);
+        }
+        if (comm->node_roots_comm) {
+            propagate_subcomms(comm, comm->node_roots_comm);
+        }
     }
 
   fn_exit:
@@ -748,35 +835,6 @@ int MPIR_Comm_commit(MPIR_Comm * comm)
     goto fn_exit;
 }
 
-/* Returns true if the given communicator is aware of node topology information,
-   false otherwise.  Such information could be used to implement more efficient
-   collective communication, for example. */
-int MPIR_Comm_is_parent_comm(MPIR_Comm * comm)
-{
-    return (comm->hierarchy_kind == MPIR_COMM_HIERARCHY_KIND__PARENT);
-}
-
-/* Returns true if the communicator is node-aware and processes in all the nodes
-   are consecutive. For example, if node 0 contains "0, 1, 2, 3", node 1
-   contains "4, 5, 6", and node 2 contains "7", we shall return true. */
-int MPII_Comm_is_node_consecutive(MPIR_Comm * comm)
-{
-    int i = 0, curr_nodeidx = 0;
-    int *internode_table = comm->internode_table;
-
-    if (!MPIR_Comm_is_parent_comm(comm))
-        return 0;
-
-    for (; i < comm->local_size; i++) {
-        if (internode_table[i] == curr_nodeidx + 1)
-            curr_nodeidx++;
-        else if (internode_table[i] != curr_nodeidx)
-            return 0;
-    }
-
-    return 1;
-}
-
 /* Duplicate a communicator without copying the streams. This is the common
  * part called by both MPIR_Comm_dup_with_info_impl and MPIR_Stream_comm_create_impl.
  */
@@ -976,7 +1034,6 @@ int MPII_Comm_copy_data(MPIR_Comm * comm_ptr, MPIR_Info * info, MPIR_Comm ** out
     *outcomm_ptr = newcomm_ptr;
 
     newcomm_ptr->vcis_enabled = comm_ptr->vcis_enabled;
-    propagate_vcis_enabled(newcomm_ptr);
 
   fn_fail:
     MPIR_FUNC_EXIT;
@@ -1001,6 +1058,7 @@ int MPIR_Comm_delete_internal(MPIR_Comm * comm_ptr)
     MPIR_FUNC_ENTER;
 
     MPIR_Assert(MPIR_Object_get_ref(comm_ptr) == 0);    /* sanity check */
+    MPIR_Assert(!(comm_ptr->attr & MPIR_COMM_ATTR__SUBCOMM));
 
     /* Remove the attributes, executing the attribute delete routine.
      * Do this only if the attribute functions are defined.
@@ -1089,9 +1147,9 @@ int MPIR_Comm_delete_internal(MPIR_Comm * comm_ptr)
 
         /* free the intra/inter-node communicators, if they exist */
         if (comm_ptr->node_comm)
-            MPIR_Comm_release(comm_ptr->node_comm);
+            MPIR_Subcomm_free(comm_ptr->node_comm);
         if (comm_ptr->node_roots_comm)
-            MPIR_Comm_release(comm_ptr->node_roots_comm);
+            MPIR_Subcomm_free(comm_ptr->node_roots_comm);
         MPL_free(comm_ptr->intranode_table);
         MPL_free(comm_ptr->internode_table);
 
@@ -1199,52 +1257,6 @@ int MPII_collect_info_key(MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, const char
     goto fn_exit;
 }
 
-/* Returns true if the communicator is node-aware and the number of processes in all the nodes are
- * same */
-int MPII_Comm_is_node_balanced(MPIR_Comm * comm, int *num_nodes, bool * node_balanced)
-{
-    int i = 0;
-    int mpi_errno = MPI_SUCCESS;
-    int *ranks_per_node;
-    *num_nodes = 0;
-
-    MPIR_CHKPMEM_DECL();
-
-    if (!MPIR_Comm_is_parent_comm(comm)) {
-        *node_balanced = false;
-        goto fn_exit;
-    }
-
-    /* Find maximum value in the internode_table */
-    for (i = 0; i < comm->local_size; i++) {
-        if (comm->internode_table[i] > *num_nodes) {
-            *num_nodes = comm->internode_table[i];
-        }
-    }
-    /* number of nodes is max_node_id + 1 */
-    (*num_nodes)++;
-
-    MPIR_CHKPMEM_CALLOC(ranks_per_node, *num_nodes * sizeof(int), MPL_MEM_OTHER);
-
-    for (i = 0; i < comm->local_size; i++) {
-        ranks_per_node[comm->internode_table[i]]++;
-    }
-
-    for (i = 1; i < *num_nodes; i++) {
-        if (ranks_per_node[i - 1] != ranks_per_node[i]) {
-            *node_balanced = false;
-            goto fn_exit;
-        }
-    }
-
-    *node_balanced = true;
-  fn_exit:
-    MPIR_CHKPMEM_REAP();
-    return mpi_errno;
-  fn_fail:
-    goto fn_exit;
-}
-
 int MPIR_Comm_save_inactive_request(MPIR_Comm * comm, MPIR_Request * request)
 {
     MPID_THREAD_CS_ENTER(VCI, comm->mutex);
diff --git a/src/mpi/init/mpir_init.c b/src/mpi/init/mpir_init.c
index 3a817a1804f..a9675c5d518 100644
--- a/src/mpi/init/mpir_init.c
+++ b/src/mpi/init/mpir_init.c
@@ -278,15 +278,20 @@ int MPII_Init_thread(int *argc, char ***argv, int user_required, int *provided,
     need_init_builtin_comms = is_world_model;
 #endif
     if (need_init_builtin_comms) {
-        mpi_errno = MPIR_init_comm_world();
-        MPIR_ERR_CHECK(mpi_errno);
-
-        mpi_errno = MPIR_init_comm_self();
-        MPIR_ERR_CHECK(mpi_errno);
+        if (!MPIR_Process.comm_world) {
+            mpi_errno = MPIR_init_comm_world();
+            MPIR_ERR_CHECK(mpi_errno);
+        }
 
+        if (!MPIR_Process.comm_self) {
+            mpi_errno = MPIR_init_comm_self();
+            MPIR_ERR_CHECK(mpi_errno);
+        }
 #ifdef MPID_NEEDS_ICOMM_WORLD
-        mpi_errno = MPIR_init_icomm_world();
-        MPIR_ERR_CHECK(mpi_errno);
+        if (!MPIR_Process.icomm_world) {
+            mpi_errno = MPIR_init_icomm_world();
+            MPIR_ERR_CHECK(mpi_errno);
+        }
 #endif
     }
 
diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c b/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c
index bc090f12626..c96007b4e7b 100644
--- a/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c
+++ b/src/mpid/ch3/channels/nemesis/src/ch3_rma_shm.c
@@ -35,7 +35,7 @@ int MPIDI_CH3_SHM_Win_shared_query(MPIR_Win * win_ptr, int target_rank, MPI_Aint
 
         for (i = 0; i < comm_size; i++) {
             if (win_ptr->basic_info_table[i].size > 0) {
-                int local_i = win_ptr->comm_ptr->intranode_table[i];
+                int local_i = MPIR_Get_intranode_rank(win_ptr->comm_ptr, i);
                 MPIR_Assert(local_i >= 0 && local_i < win_ptr->comm_ptr->node_comm->local_size);
                 *size = win_ptr->basic_info_table[i].size;
                 *disp_unit = win_ptr->basic_info_table[i].disp_unit;
@@ -46,7 +46,7 @@ int MPIDI_CH3_SHM_Win_shared_query(MPIR_Win * win_ptr, int target_rank, MPI_Aint
 
     }
     else {
-        int local_target_rank = win_ptr->comm_ptr->intranode_table[target_rank];
+        int local_target_rank = MPIR_Get_intranode_rank(win_ptr->comm_ptr, target_rank);
         if (local_target_rank >= 0 && win_ptr->shm_base_addrs) {
             MPIR_Assert(local_target_rank < win_ptr->comm_ptr->node_comm->local_size);
             *size = win_ptr->basic_info_table[target_rank].size;
diff --git a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c
index a2e01b218c4..843781e89e9 100644
--- a/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c
+++ b/src/mpid/ch3/channels/nemesis/src/ch3_win_fns.c
@@ -206,7 +206,7 @@ static int MPIDI_CH3I_SHM_Wins_match(MPIR_Win ** win_ptr, MPIR_Win ** matched_wi
 
         base_diff = 0;
         for (i = 0; i < comm_size; ++i) {
-            int i_node_rank = (*win_ptr)->comm_ptr->intranode_table[i];
+            int i_node_rank = MPIR_Get_intranode_rank((*win_ptr)->comm_ptr, i);
             if (i_node_rank >= 0) {
                 MPIR_Assert(i_node_rank < node_size);
 
diff --git a/src/mpid/ch3/include/mpid_rma_shm.h b/src/mpid/ch3/include/mpid_rma_shm.h
index eea0b6ba18e..2c7f2b8a94d 100644
--- a/src/mpid/ch3/include/mpid_rma_shm.h
+++ b/src/mpid/ch3/include/mpid_rma_shm.h
@@ -108,7 +108,7 @@ static inline int MPIDI_CH3I_Shm_put_op(const void *origin_addr, MPI_Aint origin
     MPIR_FUNC_ENTER;
 
     if (win_ptr->shm_allocated == TRUE) {
-        int local_target_rank = win_ptr->comm_ptr->intranode_table[target_rank];
+        int local_target_rank = MPIR_Get_intranode_rank(win_ptr->comm_ptr, target_rank);
         MPIR_Assert(local_target_rank >= 0);
         base = win_ptr->shm_base_addrs[local_target_rank];
         disp_unit = win_ptr->basic_info_table[target_rank].disp_unit;
@@ -151,7 +151,7 @@ static inline int MPIDI_CH3I_Shm_acc_op(const void *origin_addr, MPI_Aint origin
     MPIR_FUNC_ENTER;
 
     if (win_ptr->shm_allocated == TRUE) {
-        int local_target_rank = win_ptr->comm_ptr->intranode_table[target_rank];
+        int local_target_rank = MPIR_Get_intranode_rank(win_ptr->comm_ptr, target_rank);
         MPIR_Assert(local_target_rank >= 0);
         shm_op = 1;
         base = win_ptr->shm_base_addrs[local_target_rank];
@@ -268,7 +268,7 @@ static inline int MPIDI_CH3I_Shm_get_acc_op(const void *origin_addr, MPI_Aint or
         is_empty_origin = TRUE;
 
     if (win_ptr->shm_allocated == TRUE) {
-        int local_target_rank = win_ptr->comm_ptr->intranode_table[target_rank];
+        int local_target_rank = MPIR_Get_intranode_rank(win_ptr->comm_ptr, target_rank);
         MPIR_Assert(local_target_rank >= 0);
         base = win_ptr->shm_base_addrs[local_target_rank];
         disp_unit = win_ptr->basic_info_table[target_rank].disp_unit;
@@ -375,7 +375,7 @@ static inline int MPIDI_CH3I_Shm_get_op(void *origin_addr, MPI_Aint origin_count
     MPIR_FUNC_ENTER;
 
     if (win_ptr->shm_allocated == TRUE) {
-        int local_target_rank = win_ptr->comm_ptr->intranode_table[target_rank];
+        int local_target_rank = MPIR_Get_intranode_rank(win_ptr->comm_ptr, target_rank);
         MPIR_Assert(local_target_rank >= 0);
         base = win_ptr->shm_base_addrs[local_target_rank];
         disp_unit = win_ptr->basic_info_table[target_rank].disp_unit;
@@ -412,7 +412,7 @@ static inline int MPIDI_CH3I_Shm_cas_op(const void *origin_addr, const void *com
     MPIR_FUNC_ENTER;
 
     if (win_ptr->shm_allocated == TRUE) {
-        int local_target_rank = win_ptr->comm_ptr->intranode_table[target_rank];
+        int local_target_rank = MPIR_Get_intranode_rank(win_ptr->comm_ptr, target_rank);
         MPIR_Assert(local_target_rank >= 0);
         base = win_ptr->shm_base_addrs[local_target_rank];
         disp_unit = win_ptr->basic_info_table[target_rank].disp_unit;
@@ -463,7 +463,7 @@ static inline int MPIDI_CH3I_Shm_fop_op(const void *origin_addr, void *result_ad
     }
 
     if (win_ptr->shm_allocated == TRUE) {
-        int local_target_rank = win_ptr->comm_ptr->intranode_table[target_rank];
+        int local_target_rank = MPIR_Get_intranode_rank(win_ptr->comm_ptr, target_rank);
         MPIR_Assert(local_target_rank >= 0);
         base = win_ptr->shm_base_addrs[local_target_rank];
         disp_unit = win_ptr->basic_info_table[target_rank].disp_unit;
diff --git a/src/mpid/ch3/src/mpid_vc.c b/src/mpid/ch3/src/mpid_vc.c
index 9a4a7e87d7b..4dc4e1c00f1 100644
--- a/src/mpid/ch3/src/mpid_vc.c
+++ b/src/mpid/ch3/src/mpid_vc.c
@@ -705,10 +705,11 @@ char MPIU_hostname[MAX_HOSTNAME_LEN] = "_UNKNOWN_"; /* '_' is an illegal char fo
 
 int MPID_Get_node_id(MPIR_Comm *comm, int rank, int *id_p)
 {
-    if (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) {
-        *id_p = comm->dev.vcrt->vcr_table[rank]->node_id;
+    MPIR_Lpid lpid = MPIR_comm_rank_to_lpid(comm, rank);
+    if (lpid >= 0 && lpid < MPIR_Process.size) {
+        *id_p = MPIR_Process.node_map[lpid];
     } else {
-        *id_p = comm->dev.local_vcrt->vcr_table[rank]->node_id;
+        *id_p = -1;
     }
     return MPI_SUCCESS;
 }
diff --git a/src/mpid/ch4/include/mpidpre.h b/src/mpid/ch4/include/mpidpre.h
index bab94b1bcf9..5a6232d89a5 100644
--- a/src/mpid/ch4/include/mpidpre.h
+++ b/src/mpid/ch4/include/mpidpre.h
@@ -578,7 +578,6 @@ typedef struct MPIDI_Devcomm_t {
         struct MPIR_Comm *multi_leads_comm;
         /* sub communicators related for multi-leaders based implementation */
         struct MPIR_Comm *inter_node_leads_comm, *sub_node_comm, *intra_node_leads_comm;
-        int spanned_num_nodes;  /* comm spans over these number of nodes */
         /* Pointers to store info of multi-leaders based compositions */
         struct MPIDI_Multileads_comp_info_t *alltoall_comp_info, *allgather_comp_info,
             *allreduce_comp_info;
diff --git a/src/mpid/ch4/shm/posix/eager/iqueue/iqueue_init.c b/src/mpid/ch4/shm/posix/eager/iqueue/iqueue_init.c
index 6ee422cb4e3..dc341a9f0c2 100644
--- a/src/mpid/ch4/shm/posix/eager/iqueue/iqueue_init.c
+++ b/src/mpid/ch4/shm/posix/eager/iqueue/iqueue_init.c
@@ -164,10 +164,11 @@ int MPIDI_POSIX_iqueue_set_vcis(MPIR_Comm * comm, int max_vcis)
 
     MPIDI_POSIX_eager_iqueue_global.max_vcis = max_vcis;
 
-    if (comm->node_comm) {
-        mpi_errno = MPIR_Barrier_impl(comm->node_comm, MPIR_ERR_NONE);
-        MPIR_ERR_CHECK(mpi_errno);
-    }
+    MPIR_Comm *node_comm = MPIR_Comm_get_node_comm(comm);
+    MPIR_Assert(node_comm);
+
+    mpi_errno = MPIR_Barrier_impl(node_comm, MPIR_ERR_NONE);
+    MPIR_ERR_CHECK(mpi_errno);
 
   fn_exit:
     return mpi_errno;
diff --git a/src/mpid/ch4/shm/posix/posix_vci.c b/src/mpid/ch4/shm/posix/posix_vci.c
index ccdeb9fcae8..c9c568e911a 100644
--- a/src/mpid/ch4/shm/posix/posix_vci.c
+++ b/src/mpid/ch4/shm/posix/posix_vci.c
@@ -13,15 +13,13 @@ int MPIDI_POSIX_comm_set_vcis(MPIR_Comm * comm, int num_vcis)
     /* We only set up vcis once */
     MPIR_Assert(MPIDI_POSIX_global.num_vcis == 1);
 
+    MPIR_Comm *node_comm = MPIR_Comm_get_node_comm(comm);
+    MPIR_Assert(node_comm);
+
     int max_vcis;
-    MPIR_Comm *node_comm = comm->node_comm;
-    if (node_comm == NULL) {
-        max_vcis = num_vcis;
-    } else {
-        mpi_errno = MPIR_Allreduce_impl(&num_vcis, &max_vcis, 1, MPIR_INT_INTERNAL, MPI_MAX,
-                                        node_comm, MPIR_ERR_NONE);
-        MPIR_ERR_CHECK(mpi_errno);
-    }
+    mpi_errno = MPIR_Allreduce_impl(&num_vcis, &max_vcis, 1, MPIR_INT_INTERNAL, MPI_MAX,
+                                    node_comm, MPIR_ERR_NONE);
+    MPIR_ERR_CHECK(mpi_errno);
 
     if (max_vcis > 1) {
         for (int i = 1; i < max_vcis; i++) {
diff --git a/src/mpid/ch4/src/ch4_coll.h b/src/mpid/ch4/src/ch4_coll.h
index 5b586f76c09..e579e051827 100644
--- a/src/mpid/ch4/src/ch4_coll.h
+++ b/src/mpid/ch4/src/ch4_coll.h
@@ -148,8 +148,7 @@ MPL_STATIC_INLINE_PREFIX int MPID_Barrier(MPIR_Comm * comm, MPIR_Errflag_t errfl
         case 1:
             MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank,
                                            (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) &&
-                                           (comm->hierarchy_kind ==
-                                            MPIR_COMM_HIERARCHY_KIND__PARENT), mpi_errno,
+                                           MPIR_Comm_is_parent_comm(comm), mpi_errno,
                                            "Barrier composition alpha cannot be applied.\n");
             mpi_errno = MPIDI_Barrier_intra_composition_alpha(comm, errflag);
             break;
@@ -255,8 +254,7 @@ MPL_STATIC_INLINE_PREFIX int MPID_Bcast(void *buffer, MPI_Aint count, MPI_Dataty
         case 1:
             MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank,
                                            (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) &&
-                                           (comm->hierarchy_kind ==
-                                            MPIR_COMM_HIERARCHY_KIND__PARENT), mpi_errno,
+                                           MPIR_Comm_is_parent_comm(comm), mpi_errno,
                                            "Bcast composition alpha cannot be applied.\n");
             mpi_errno =
                 MPIDI_Bcast_intra_composition_alpha(buffer, count, datatype, root, comm, errflag);
@@ -264,8 +262,7 @@ MPL_STATIC_INLINE_PREFIX int MPID_Bcast(void *buffer, MPI_Aint count, MPI_Dataty
         case 2:
             MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank,
                                            (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) &&
-                                           (comm->hierarchy_kind ==
-                                            MPIR_COMM_HIERARCHY_KIND__PARENT), mpi_errno,
+                                           MPIR_Comm_is_parent_comm(comm), mpi_errno,
                                            "Bcast composition beta cannot be applied.\n");
             mpi_errno =
                 MPIDI_Bcast_intra_composition_beta(buffer, count, datatype, root, comm, errflag);
@@ -280,8 +277,7 @@ MPL_STATIC_INLINE_PREFIX int MPID_Bcast(void *buffer, MPI_Aint count, MPI_Dataty
         case 4:
             MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank,
                                            (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) &&
-                                           (comm->hierarchy_kind ==
-                                            MPIR_COMM_HIERARCHY_KIND__PARENT), mpi_errno,
+                                           MPIR_Comm_is_parent_comm(comm), mpi_errno,
                                            "Bcast composition delta cannot be applied.\n");
             mpi_errno =
                 MPIDI_Bcast_intra_composition_delta(buffer, count, datatype, root, comm, errflag);
@@ -311,9 +307,6 @@ MPL_STATIC_INLINE_PREFIX int MPID_Bcast(void *buffer, MPI_Aint count, MPI_Dataty
 
 MPL_STATIC_INLINE_PREFIX void MPIDI_Allreduce_fill_multi_leads_info(MPIR_Comm * comm)
 {
-    int node_comm_size = 0, num_nodes;
-    bool node_balanced = false;
-
     if (MPIDI_COMM(comm, allreduce_comp_info) == NULL) {
         /* If multi-leads allreduce is enabled and the info object is not yet set */
         MPIDI_COMM(comm, allreduce_comp_info) =
@@ -325,13 +318,7 @@ MPL_STATIC_INLINE_PREFIX void MPIDI_Allreduce_fill_multi_leads_info(MPIR_Comm *
     }
     /* Find if the comm meets the constraints and store that info in the data structure */
     if (MPIDI_COMM_ALLREDUCE(comm, use_multi_leads) == -1) {
-        if (comm->node_comm)
-            node_comm_size = MPIR_Comm_size(comm->node_comm);
-        /* Get number of nodes it spans over and if it has equal number of ranks per node */
-        MPII_Comm_is_node_balanced(comm, &num_nodes, &node_balanced);
-        MPIDI_COMM(comm, spanned_num_nodes) = num_nodes;
-
-        if (num_nodes > 1 && node_comm_size > 1 && node_balanced) {
+        if (MPII_Comm_is_node_balanced(comm)) {
             MPIDI_COMM_ALLREDUCE(comm, use_multi_leads) = 1;
         } else {
             MPIDI_COMM_ALLREDUCE(comm, use_multi_leads) = 0;
@@ -435,8 +422,7 @@ MPL_STATIC_INLINE_PREFIX int MPID_Allreduce(const void *sendbuf, void *recvbuf,
         case 1:
             MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank,
                                            (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) &&
-                                           (comm->hierarchy_kind ==
-                                            MPIR_COMM_HIERARCHY_KIND__PARENT) &&
+                                           MPIR_Comm_is_parent_comm(comm) &&
                                            is_commutative, mpi_errno,
                                            "Allreduce composition alpha cannot be applied.\n");
             mpi_errno =
@@ -515,9 +501,6 @@ MPL_STATIC_INLINE_PREFIX int MPID_Allreduce(const void *sendbuf, void *recvbuf,
 
 MPL_STATIC_INLINE_PREFIX void MPIDI_Allgather_fill_multi_leads_info(MPIR_Comm * comm)
 {
-    int node_comm_size = 0, num_nodes;
-    bool node_balanced = false;
-
     if (MPIDI_COMM(comm, allgather_comp_info) == NULL) {
         MPIDI_COMM(comm, allgather_comp_info) =
             MPL_malloc(sizeof(MPIDI_Multileads_comp_info_t), MPL_MEM_OTHER);
@@ -527,14 +510,7 @@ MPL_STATIC_INLINE_PREFIX void MPIDI_Allgather_fill_multi_leads_info(MPIR_Comm *
     }
     /* Find if the comm meets the constraints and store that info in the data structure */
     if (MPIDI_COMM_ALLGATHER(comm, use_multi_leads) == -1) {
-        if (comm->node_comm)
-            node_comm_size = MPIR_Comm_size(comm->node_comm);
-        /* Get number of nodes it spans over and if it has equal number of ranks per node */
-        MPII_Comm_is_node_balanced(comm, &num_nodes, &node_balanced);
-        MPIDI_COMM(comm, spanned_num_nodes) = num_nodes;
-
-        if (num_nodes > 1 && node_comm_size > 1 && node_balanced &&
-            MPII_Comm_is_node_consecutive(comm))
+        if (MPII_Comm_is_node_canonical(comm))
             MPIDI_COMM_ALLGATHER(comm, use_multi_leads) = 1;
         else
             MPIDI_COMM_ALLGATHER(comm, use_multi_leads) = 0;
@@ -967,9 +943,6 @@ MPL_STATIC_INLINE_PREFIX int MPID_Gatherv(const void *sendbuf, MPI_Aint sendcoun
 
 MPL_STATIC_INLINE_PREFIX void MPIDI_Alltoall_fill_multi_leads_info(MPIR_Comm * comm)
 {
-    int node_comm_size = 0, num_nodes;
-    bool node_balanced = false;
-
     if (MPIDI_COMM(comm, alltoall_comp_info) == NULL) {
         MPIDI_COMM(comm, alltoall_comp_info) =
             MPL_malloc(sizeof(MPIDI_Multileads_comp_info_t), MPL_MEM_OTHER);
@@ -979,14 +952,7 @@ MPL_STATIC_INLINE_PREFIX void MPIDI_Alltoall_fill_multi_leads_info(MPIR_Comm * c
     }
     /* Find if the comm meets the constraints and store that info in the data structure */
     if (MPIDI_COMM_ALLTOALL(comm, use_multi_leads) == -1) {
-        if (comm->node_comm)
-            node_comm_size = MPIR_Comm_size(comm->node_comm);
-        /* Get number of nodes it spans over and if it has equal number of ranks per node */
-        MPII_Comm_is_node_balanced(comm, &num_nodes, &node_balanced);
-        MPIDI_COMM(comm, spanned_num_nodes) = num_nodes;
-
-        if (num_nodes > 1 && node_comm_size > 1 && node_balanced &&
-            MPII_Comm_is_node_consecutive(comm))
+        if (MPII_Comm_is_node_canonical(comm))
             MPIDI_COMM_ALLTOALL(comm, use_multi_leads) = 1;
         else
             MPIDI_COMM_ALLTOALL(comm, use_multi_leads) = 0;
@@ -1329,8 +1295,7 @@ MPL_STATIC_INLINE_PREFIX int MPID_Reduce(const void *sendbuf, void *recvbuf,
     switch (MPIR_CVAR_REDUCE_COMPOSITION) {
         case 1:
             MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, comm->comm_kind == MPIR_COMM_KIND__INTRACOMM
-                                           && comm->hierarchy_kind ==
-                                           MPIR_COMM_HIERARCHY_KIND__PARENT &&
+                                           && MPIR_Comm_is_parent_comm(comm) &&
                                            MPIR_Op_is_commutative(op), mpi_errno,
                                            "Reduce composition alpha cannot be applied.\n");
             mpi_errno =
@@ -1339,8 +1304,7 @@ MPL_STATIC_INLINE_PREFIX int MPID_Reduce(const void *sendbuf, void *recvbuf,
             break;
         case 2:
             MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, comm->comm_kind == MPIR_COMM_KIND__INTRACOMM
-                                           && comm->hierarchy_kind ==
-                                           MPIR_COMM_HIERARCHY_KIND__PARENT &&
+                                           && MPIR_Comm_is_parent_comm(comm) &&
                                            MPIR_Op_is_commutative(op), mpi_errno,
                                            "Reduce composition beta cannot be applied.\n");
             mpi_errno =
diff --git a/src/mpid/ch4/src/ch4_coll_impl.h b/src/mpid/ch4/src/ch4_coll_impl.h
index d5389a6dd3e..e565187f2a0 100644
--- a/src/mpid/ch4/src/ch4_coll_impl.h
+++ b/src/mpid/ch4/src/ch4_coll_impl.h
@@ -1170,7 +1170,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Alltoall_intra_composition_alpha(const void *
         MPIR_Datatype_get_size_macro(recvtype, type_size);
     }
 
-    num_nodes = MPIDI_COMM(comm_ptr, spanned_num_nodes);
+    num_nodes = comm_ptr->num_external;
     if (sendbuf == MPI_IN_PLACE) {
         sendbuf = recvbuf;
         sendcount = recvcount;
diff --git a/src/mpid/ch4/src/ch4_comm.c b/src/mpid/ch4/src/ch4_comm.c
index f33417e3a10..6940673f476 100644
--- a/src/mpid/ch4/src/ch4_comm.c
+++ b/src/mpid/ch4/src/ch4_comm.c
@@ -135,7 +135,6 @@ int MPID_Comm_commit_pre_hook(MPIR_Comm * comm)
     MPIDI_COMM(comm, inter_node_leads_comm) = NULL;
     MPIDI_COMM(comm, sub_node_comm) = NULL;
     MPIDI_COMM(comm, intra_node_leads_comm) = NULL;
-    MPIDI_COMM(comm, spanned_num_nodes) = -1;
     MPIDI_COMM(comm, alltoall_comp_info) = NULL;
     MPIDI_COMM(comm, allgather_comp_info) = NULL;
     MPIDI_COMM(comm, allreduce_comp_info) = NULL;
@@ -221,7 +220,7 @@ int MPID_Comm_free_hook(MPIR_Comm * comm)
     MPIR_FUNC_ENTER;
 
     if (MPIDI_COMM(comm, multi_leads_comm) != NULL) {
-        MPIR_Comm_release(MPIDI_COMM(comm, multi_leads_comm));
+        MPIR_Subcomm_free(MPIDI_COMM(comm, multi_leads_comm));
     }
 
     if (MPIDI_COMM(comm, inter_node_leads_comm) != NULL) {
@@ -700,113 +699,38 @@ int MPID_Create_intercomm_from_lpids(MPIR_Comm * newcomm_ptr, int size, const MP
 }
 
 /* Create multi-leaders communicator */
-/* Create a comm with rank 0 of each node. A comm with rank 1 of each node and so on. Since these
+/* This routines assume a balanced and consecutive comm.
+ * Create a comm with rank 0 of each node. A comm with rank 1 of each node and so on. Since these
  * new comms do no overlap, it uses the same context id */
 int MPIDI_Comm_create_multi_leaders(MPIR_Comm * comm)
 {
     int mpi_errno = MPI_SUCCESS;
-    int num_local = -1, num_external = -1;
-    int local_rank = -1, external_rank = -1, rank;
-    int i = 0;
-    int *local_procs = NULL, *external_procs = NULL;
-    int *intranode_table = NULL, *internode_table = NULL;
+    MPIR_CHKLMEM_DECL();
     MPIR_FUNC_TERSE_ENTER;
 
-    mpi_errno = MPIR_Find_local(comm, &num_local, &local_rank, &local_procs, &intranode_table);
-    if (mpi_errno) {
-        if (MPIR_Err_is_fatal(mpi_errno))
-            MPIR_ERR_POP(mpi_errno);
-
-        MPL_DBG_MSG_P(MPIR_DBG_COMM, VERBOSE,
-                      "MPIR_Find_local_and_external failed for comm_ptr=%p", comm);
-        if (intranode_table)
-            MPL_free(intranode_table);
-
-        mpi_errno = MPI_SUCCESS;
-        goto fn_exit;
-    }
-
-    mpi_errno = MPIR_Find_external(comm, &num_external, &external_rank, &external_procs,
-                                   &internode_table);
-    if (mpi_errno) {
-        if (MPIR_Err_is_fatal(mpi_errno))
-            MPIR_ERR_POP(mpi_errno);
+    int num_local = comm->num_local;
+    int num_external = comm->num_external;
+    int local_rank = comm->local_rank;
+    int external_rank = comm->external_rank;
 
-        MPL_DBG_MSG_P(MPIR_DBG_COMM, VERBOSE,
-                      "MPIR_Find_local_and_external failed for comm_ptr=%p", comm);
-        if (internode_table)
-            MPL_free(internode_table);
+    /* balanced and node-consecutive */
+    MPIR_Assert(comm->local_size == num_local * num_external);
+    MPIR_Assert(local_rank == comm->rank % num_local);
+    MPIR_Assert(external_rank = comm->rank / num_local);
 
-        mpi_errno = MPI_SUCCESS;
-        goto fn_exit;
-    }
-
-    MPIR_Assert(num_local > 0);
-    MPIR_Assert(num_local > 1 || external_rank >= 0);
-    MPIR_Assert(external_rank < 0 || external_procs != NULL);
-    rank = MPIR_Comm_rank(comm);
-
-    external_rank = comm->internode_table[rank];
-    for (i = 0; i < num_external; ++i) {
+    int *external_procs;
+    MPIR_CHKLMEM_MALLOC(external_procs, num_external * sizeof(int));
+    for (int i = 0; i < num_external; ++i) {
         external_procs[i] = i * num_local + local_rank;
     }
 
-    for (i = 0; i < num_local; i++) {
-        if (local_rank == i) {
-            mpi_errno = MPIR_Comm_create(&MPIDI_COMM(comm, multi_leads_comm));
-            if (mpi_errno)
-                MPIR_ERR_POP(mpi_errno);
-
-            MPIDI_COMM(comm, multi_leads_comm)->context_id =
-                comm->context_id + MPIR_CONTEXT_MULTILEADS_OFFSET;
-            MPIDI_COMM(comm, multi_leads_comm)->recvcontext_id =
-                MPIDI_COMM(comm, multi_leads_comm)->context_id;
-            MPIDI_COMM(comm, multi_leads_comm)->rank = internode_table[rank];
-            MPIDI_COMM(comm, multi_leads_comm)->comm_kind = MPIR_COMM_KIND__INTRACOMM;
-            MPIDI_COMM(comm, multi_leads_comm)->hierarchy_kind =
-                MPIR_COMM_HIERARCHY_KIND__MULTI_LEADS;
-            MPIDI_COMM(comm, multi_leads_comm)->local_comm = NULL;
-            MPIDI_COMM(comm, multi_leads_comm)->node_comm = NULL;
-            MPIDI_COMM(comm, multi_leads_comm)->node_roots_comm = NULL;
-            MPL_DBG_MSG_D(MPIR_DBG_COMM, VERBOSE, "Create multi-leaders_comm=%p\n",
-                          MPIDI_COMM(comm, multi_leads_comm));
-
-            MPIDI_COMM(comm, multi_leads_comm)->local_size = num_external;
-            MPIDI_COMM(comm, multi_leads_comm)->coll.pof2 =
-                MPL_pof2(MPIDI_COMM(comm, multi_leads_comm)->local_size);
-            MPIDI_COMM(comm, multi_leads_comm)->remote_size = num_external;
-
-            mpi_errno = MPIR_Group_incl_impl(comm->local_group, num_external, external_procs,
-                                             &MPIDI_COMM(comm, multi_leads_comm)->local_group);
-            MPIR_ERR_CHECK(mpi_errno);
-
-            /* Notify device of communicator creation */
-            mpi_errno = MPID_Comm_commit_pre_hook(MPIDI_COMM(comm, multi_leads_comm));
-            if (mpi_errno)
-                MPIR_ERR_POP(mpi_errno);
-            /* don't call MPIR_Comm_commit here */
-
-            /* Create collectives-specific infrastructure */
-            mpi_errno = MPIR_Coll_comm_init(MPIDI_COMM(comm, multi_leads_comm));
-            if (mpi_errno)
-                MPIR_ERR_POP(mpi_errno);
-
-            mpi_errno = MPID_Comm_commit_post_hook(MPIDI_COMM(comm, multi_leads_comm));
-            if (mpi_errno)
-                MPIR_ERR_CHECK(mpi_errno);
-        }
-    }
+    mpi_errno = MPIR_Subcomm_create(comm, num_external, external_rank, external_procs,
+                                    MPIR_CONTEXT_MULTILEADS_OFFSET,
+                                    &MPIDI_COMM(comm, multi_leads_comm));
+    MPIR_ERR_CHECK(mpi_errno);
 
   fn_exit:
-    if (external_procs != NULL)
-        MPL_free(external_procs);
-    if (local_procs != NULL)
-        MPL_free(local_procs);
-    if (intranode_table != NULL)
-        MPL_free(intranode_table);
-    if (internode_table != NULL)
-        MPL_free(internode_table);
-
+    MPIR_CHKLMEM_FREEALL();
     MPIR_FUNC_TERSE_EXIT;
     return mpi_errno;
   fn_fail:
diff --git a/src/mpid/ch4/src/ch4_impl.h b/src/mpid/ch4/src/ch4_impl.h
index 190285fe88b..bf68132c93c 100644
--- a/src/mpid/ch4/src/ch4_impl.h
+++ b/src/mpid/ch4/src/ch4_impl.h
@@ -923,9 +923,9 @@ MPL_STATIC_INLINE_PREFIX int MPIDIU_win_rank_to_intra_rank(MPIR_Win * win, int r
                                                            MPIDI_winattr_t winattr)
 {
     if (winattr & MPIDI_WINATTR_DIRECT_INTRA_COMM)
-        return MPIR_Process.comm_world->intranode_table[rank];
+        return MPIR_Get_intranode_rank(MPIR_Process.comm_world, rank);
     else
-        return win->comm_ptr->intranode_table[rank];
+        return MPIR_Get_intranode_rank(win->comm_ptr, rank);
 }
 
 /* Wait until active message acc ops are done. */
diff --git a/src/mpid/ch4/src/mpidig_win.h b/src/mpid/ch4/src/mpidig_win.h
index 28bc9c07f6d..e76de9b8387 100644
--- a/src/mpid/ch4/src/mpidig_win.h
+++ b/src/mpid/ch4/src/mpidig_win.h
@@ -562,7 +562,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDIG_win_shared_query_part(MPIR_Win * win, int ra
         *disp_unit = 0;
         *((void **) baseptr) = NULL;
     } else {
-        /* find shm_rank in node_comm. Q: can we rely on comm_ptr->intranode_table? */
+        /* find shm_rank in node_comm. Q: can we rely on MPIR_Get_intranode_rank(comm_ptr, rank)? */
         MPIR_Lpid lpid = MPIR_comm_rank_to_lpid(win->comm_ptr, rank);
         int shm_rank = MPIR_Group_lpid_to_rank(win->comm_ptr->node_comm->local_group, lpid);
         MPIR_Assert(shm_rank >= 0);
diff --git a/src/mpid/common/hcoll/hcoll_init.c b/src/mpid/common/hcoll/hcoll_init.c
index 34a072d9fed..3ea812480c8 100644
--- a/src/mpid/common/hcoll/hcoll_init.c
+++ b/src/mpid/common/hcoll/hcoll_init.c
@@ -149,8 +149,7 @@ int hcoll_comm_create(MPIR_Comm * comm_ptr, void *param)
     }
     num_ranks = comm_ptr->local_size;
     if ((MPIR_COMM_KIND__INTRACOMM != comm_ptr->comm_kind) || (2 > num_ranks)
-        || comm_ptr->hierarchy_kind == MPIR_COMM_HIERARCHY_KIND__NODE_ROOTS
-        || comm_ptr->hierarchy_kind == MPIR_COMM_HIERARCHY_KIND__NODE) {
+        || (comm_ptr->attr & MPIR_COMM_ATTR__SUBCOMM)) {
         comm_ptr->hcoll_priv.is_hcoll_init = 0;
         goto fn_exit;
     }
diff --git a/src/util/Makefile.mk b/src/util/Makefile.mk
index 82a625858bc..dc7fb859a39 100644
--- a/src/util/Makefile.mk
+++ b/src/util/Makefile.mk
@@ -10,7 +10,6 @@ mpi_core_sources +=   \
     src/util/mpir_nodemap.c    \
     src/util/mpir_handlemem.c  \
     src/util/mpir_strerror.c   \
-    src/util/mpir_localproc.c  \
     src/util/mpir_netloc.c     \
     src/util/mpir_hwtopo.c     \
     src/util/mpir_nettopo.c    \
diff --git a/src/util/mpir_localproc.c b/src/util/mpir_localproc.c
deleted file mode 100644
index 1723f96c5cf..00000000000
--- a/src/util/mpir_localproc.c
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Copyright (C) by Argonne National Laboratory
- *     See COPYRIGHT in top-level directory
- */
-
-#include "mpiimpl.h"
-
-#if defined(HAVE_LIMITS_H)
-#include <limits.h>
-#endif
-#if defined(HAVE_UNISTD_H)
-#include <unistd.h>
-#endif
-#if defined(HAVE_ERRNO_H)
-#include <errno.h>
-#endif
-
-/* MPIR_Find_local  -- from the list of processes in comm,
- * builds a list of local processes, i.e., processes on this same node.
- *
- * Note that this will not work correctly for spawned or attached
- * processes.
- *
- *  OUT:
- *    local_size_p      - number of processes on this node.
- *    local_rank_p      - rank of this processes among local processes.
- *    local_ranks_p     - (*local_ranks_p)[i]     = the rank in comm
- *                        of the process with local rank i.
- *                        This is of size (*local_size_p).
- *    intranode_table_p - (*intranode_table_p)[i] = the rank in
- *    (optional)          *local_ranks_p of rank i in comm or -1 if not
- *                        applicable.  It is of size comm->remote_size.
- *                        No return if NULL is specified.
- */
-int MPIR_Find_local(MPIR_Comm * comm, int *local_size_p, int *local_rank_p,
-                    int **local_ranks_p, int **intranode_table_p)
-{
-    int mpi_errno = MPI_SUCCESS;
-    int i, local_size, local_rank;
-    int *local_ranks = NULL, *intranode_table = NULL;
-    int node_id = -1, my_node_id = -1;
-
-    MPIR_CHKPMEM_DECL();
-
-    /* local_ranks will be realloc'ed later to the appropriate size (currently unknown) */
-    /* FIXME: realloc doesn't guarantee that the allocated area will be
-     * shrunk - so using realloc is not an appropriate strategy. */
-    MPIR_CHKPMEM_MALLOC(local_ranks, sizeof(int) * comm->remote_size, MPL_MEM_COMM);
-    MPIR_CHKPMEM_MALLOC(intranode_table, sizeof(int) * comm->remote_size, MPL_MEM_COMM);
-
-    for (i = 0; i < comm->remote_size; ++i)
-        intranode_table[i] = -1;
-
-    mpi_errno = MPID_Get_node_id(comm, comm->rank, &my_node_id);
-    MPIR_ERR_CHECK(mpi_errno);
-    MPIR_Assert(my_node_id >= 0);
-
-    local_size = 0;
-    local_rank = -1;
-
-    /* Scan through the list of processes in comm. */
-    for (i = 0; i < comm->remote_size; ++i) {
-        mpi_errno = MPID_Get_node_id(comm, i, &node_id);
-        MPIR_ERR_CHECK(mpi_errno);
-
-        /* The upper level can catch this non-fatal error and should be
-         * able to recover gracefully. */
-        MPIR_ERR_CHKANDJUMP(node_id < 0, mpi_errno, MPI_ERR_OTHER, "**dynamic_node_ids");
-
-        /* build list of local processes */
-        if (node_id == my_node_id) {
-            if (i == comm->rank)
-                local_rank = local_size;
-
-            intranode_table[i] = local_size;
-            local_ranks[local_size] = i;
-            ++local_size;
-        }
-    }
-
-#ifdef ENABLE_DEBUG
-    printf("------------------------------------------------------------------------\n");
-    printf("[%d]comm = %p\n", comm->rank, comm);
-    printf("[%d]comm->size = %d\n", comm->rank, comm->remote_size);
-    printf("[%d]comm->rank = %d\n", comm->rank, comm->rank);
-    printf("[%d]local_size = %d\n", comm->rank, local_size);
-    printf("[%d]local_rank = %d\n", comm->rank, local_rank);
-    printf("[%d]local_ranks = %p\n", comm->rank, local_ranks);
-    for (i = 0; i < local_size; ++i)
-        printf("[%d]  local_ranks[%d] = %d\n", comm->rank, i, local_ranks[i]);
-    printf("[%d]intranode_table = %p\n", comm->rank, intranode_table);
-    for (i = 0; i < comm->remote_size; ++i)
-        printf("[%d]  intranode_table[%d] = %d\n", comm->rank, i, intranode_table[i]);
-#endif
-
-    MPIR_CHKPMEM_COMMIT();
-
-    *local_size_p = local_size;
-    *local_rank_p = local_rank;
-
-    *local_ranks_p = MPL_realloc(local_ranks, sizeof(int) * local_size, MPL_MEM_COMM);
-    MPIR_ERR_CHKANDJUMP(*local_ranks_p == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem2");
-
-    if (intranode_table_p)
-        *intranode_table_p = intranode_table;   /* no need to realloc */
-    else
-        MPL_free(intranode_table);      /* free internally if caller passes NULL */
-
-  fn_exit:
-    return mpi_errno;
-  fn_fail:
-    MPIR_CHKPMEM_REAP();
-    goto fn_exit;
-}
-
-/* MPIR_Find_external -- from the list of processes in comm,
- * builds a list of external processes, i.e., one process from each node.
- * You can think of this as the root or main process for each node.
- *
- * Note that this will return non-fatal error if there are spawned or attached
- * processes. The caller (MPIR_Comm_create_subcomms) should handle it.
- *
- *  OUT:
- *    external_size_p   - number of external processes
- *    external_rank_p   - rank of this process among the external
- *                        processes, or -1 if this process is not external
- *    external_ranks_p  - (*external_ranks_p)[i]   = the rank in comm
- *                        of the process with external rank i.
- *                        This is of size (*external_size_p)
- *    internode_table_p - (*internode_table_p)[i]  = the rank in
- *    (optional)          *external_ranks_p of the root of the node
- *                        containing rank i in comm.  It is of size
- *                        comm->remote_size. No return if NULL is specified.
- */
-int MPIR_Find_external(MPIR_Comm * comm, int *external_size_p, int *external_rank_p,
-                       int **external_ranks_p, int **internode_table_p)
-{
-    int mpi_errno = MPI_SUCCESS;
-    int *nodes;
-    int i, external_size, external_rank;
-    int *external_ranks, *internode_table;
-    int node_id;
-
-    MPIR_CHKLMEM_DECL();
-    MPIR_CHKPMEM_DECL();
-
-    /* Scan through the list of processes in comm and add one
-     * process from each node to the list of "external" processes.  We
-     * add the first process we find from each node.  nodes[] is an
-     * array where we keep track of whether we have already added that
-     * node to the list. */
-
-    /* external_ranks will be realloc'ed later to the appropriate size (currently unknown) */
-    /* FIXME: realloc doesn't guarantee that the allocated area will be
-     * shrunk - so using realloc is not an appropriate strategy. */
-    MPIR_CHKPMEM_MALLOC(external_ranks, sizeof(int) * comm->remote_size, MPL_MEM_COMM);
-    MPIR_CHKPMEM_MALLOC(internode_table, sizeof(int) * comm->remote_size, MPL_MEM_COMM);
-
-    int num_nodes = MPIR_Process.num_nodes;
-    if (MPIR_Process.node_hostnames) {
-        num_nodes = utarray_len(MPIR_Process.node_hostnames);
-    }
-    MPIR_CHKLMEM_MALLOC(nodes, sizeof(int) * num_nodes);
-
-    /* nodes maps node_id to rank in external_ranks of leader for that node */
-    for (i = 0; i < num_nodes; ++i)
-        nodes[i] = -1;
-
-    external_size = 0;
-    external_rank = -1;
-
-    for (i = 0; i < comm->remote_size; ++i) {
-        mpi_errno = MPID_Get_node_id(comm, i, &node_id);
-        MPIR_ERR_CHECK(mpi_errno);
-
-        /* The upper level can catch this non-fatal error and should be
-         * able to recover gracefully. */
-        MPIR_ERR_CHKANDJUMP(node_id < 0, mpi_errno, MPI_ERR_OTHER, "**dynamic_node_ids");
-
-        MPIR_Assert(node_id < num_nodes);
-
-        /* build list of external processes */
-        if (nodes[node_id] == -1) {
-            if (i == comm->rank)
-                external_rank = external_size;
-            nodes[node_id] = external_size;
-            external_ranks[external_size] = i;
-            ++external_size;
-        }
-
-        /* build the map from rank in comm to rank in external_ranks */
-        internode_table[i] = nodes[node_id];
-    }
-
-#ifdef ENABLE_DEBUG
-    printf("------------------------------------------------------------------------\n");
-    printf("[%d]comm = %p\n", comm->rank, comm);
-    printf("[%d]comm->size = %d\n", comm->rank, comm->remote_size);
-    printf("[%d]comm->rank = %d\n", comm->rank, comm->rank);
-    printf("[%d]external_size = %d\n", comm->rank, external_size);
-    printf("[%d]external_rank = %d\n", comm->rank, external_rank);
-    printf("[%d]external_ranks = %p\n", comm->rank, external_ranks);
-    for (i = 0; i < external_size; ++i)
-        printf("[%d]  external_ranks[%d] = %d\n", comm->rank, i, external_ranks[i]);
-    printf("[%d]internode_table = %p\n", comm->rank, internode_table);
-    for (i = 0; i < comm->remote_size; ++i)
-        printf("[%d]  internode_table[%d] = %d\n", comm->rank, i, internode_table[i]);
-    printf("[%d]nodes = %p\n", comm->rank, nodes);
-    for (i = 0; i < num_nodes; ++i)
-        printf("[%d]  nodes[%d] = %d\n", comm->rank, i, nodes[i]);
-#endif
-
-    MPIR_CHKPMEM_COMMIT();
-
-    *external_size_p = external_size;
-    *external_rank_p = external_rank;
-    *external_ranks_p = MPL_realloc(external_ranks, sizeof(int) * external_size, MPL_MEM_COMM);
-    MPIR_ERR_CHKANDJUMP(*external_ranks_p == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem2");
-
-    if (internode_table_p)
-        *internode_table_p = internode_table;   /* no need to realloc */
-    else
-        MPL_free(internode_table);      /* free internally if caller passes NULL */
-
-  fn_exit:
-    MPIR_CHKLMEM_FREEALL();
-    return mpi_errno;
-  fn_fail:
-    MPIR_CHKPMEM_REAP();
-    goto fn_exit;
-}
-
-/* maps rank r in comm_ptr to the rank of the leader for r's node in
-   comm_ptr->node_roots_comm and returns this value.
-
-   This function does NOT use mpich error handling.
- */
-int MPIR_Get_internode_rank(MPIR_Comm * comm_ptr, int r)
-{
-    int mpi_errno = MPI_SUCCESS;
-    MPIR_Comm_valid_ptr(comm_ptr, mpi_errno, TRUE);
-    MPIR_Assert(mpi_errno == MPI_SUCCESS);
-    MPIR_Assert(r < comm_ptr->remote_size);
-    MPIR_Assert(comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM);
-    MPIR_Assert(comm_ptr->internode_table != NULL);
-
-    return comm_ptr->internode_table[r];
-}
-
-/* maps rank r in comm_ptr to the rank in comm_ptr->node_comm or -1 if r is not
-   a member of comm_ptr->node_comm.
-
-   This function does NOT use mpich error handling.
- */
-int MPIR_Get_intranode_rank(MPIR_Comm * comm_ptr, int r)
-{
-    int mpi_errno = MPI_SUCCESS;
-    MPIR_Comm_valid_ptr(comm_ptr, mpi_errno, TRUE);
-    MPIR_Assert(mpi_errno == MPI_SUCCESS);
-    MPIR_Assert(r < comm_ptr->remote_size);
-    MPIR_Assert(comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM);
-    MPIR_Assert(comm_ptr->intranode_table != NULL);
-
-    /* FIXME this could/should be a list of ranks on the local node, which
-     * should take up much less space on a typical thin(ish)-node system. */
-    return comm_ptr->intranode_table[r];
-}


Reviews:
user: login: raffenet
body: A few comments but looks OK. It's going to be disruptive to anyone maintaining hierarchical algos, so we should note in CHANGES. Maybe with a nod to the benefits 😉.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-07T16:25:46Z
user: login: hzhou
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-07T16:45:32Z
user: login: hzhou
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-07T16:47:05Z
user: login: hzhou
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-07T16:50:03Z
user: login: raffenet
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-07T19:05:00Z
user: login: hzhou
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-07T19:27:27Z
user: login: raffenet
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-07T19:55:41Z
user: login: raffenet
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-07T20:23:33Z

Review comments:
diff_hunk: @@ -121,6 +121,9 @@ enum MPIR_COMM_HINT_PREDEFINED_t {
   S*/
 struct MPIR_Comm {
     MPIR_OBJECT_HEADER;         /* adds handle and ref_count fields */
+    int attr;                   /* if attr is 0, only the core set of fields are set.
path: src/include/mpir_comm.h
user: login: raffenet
body: Is there some way we can make it clearer these are separate from user attributes? It's not critical, just noting the name collision.
created_at: 2025-04-07T15:22:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -514,6 +514,28 @@ static int check_hierarchy(MPIR_Comm * comm)
     comm->node_comm = NULL;
     comm->node_roots_comm = NULL;
 
+    if (num_nodes == 1) {
+        comm->hierarchy_flags |= MPIR_COMM_HIERARCHY__SINGLE_NODE;
+    } else if (num_local == 1) {
+        comm->hierarchy_flags |= MPIR_COMM_HIERARCHY__NO_LOCAL;
+    } else {
+        bool is_node_consecutive = true;
+        for (int i = 1; i < comm_size; i++) {
+            if (internode_table[i] < internode_table[i - 1]) {
+                is_node_consecutive = false;
+                break;
+            }
+        }
+
+        if (is_node_consecutive) {
+            comm->hierarchy_flags |= MPIR_COMM_HIERARCHY__NODE_CONSECUTIVE;
+        }
+
+        if (num_local * num_nodes == comm_size) {
+            comm->hierarchy_flags |= MPIR_COMM_HIERARCHY__NODE_BALANCED;
+        }
path: src/mpi/comm/commutil.c
user: login: raffenet
body: is this check strict enough? the old check put all the ranks in node buckets and checked that each bucket was the same size.
created_at: 2025-04-07T16:11:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -344,20 +344,28 @@ MPL_STATIC_INLINE_PREFIX int MPIR_Stream_comm_set_attr(MPIR_Comm * comm, int src
 
 MPL_STATIC_INLINE_PREFIX int MPIR_Get_internode_rank(MPIR_Comm * comm, int r)
 {
-    /* TODO: optimize for special ones such as node_consecutive and balanced */
     MPIR_Assert(comm->attr | MPIR_COMM_ATTR__HIERARCHY);
-    MPIR_Assert(comm->internode_table);
-
-    return comm->internode_table[r];
+    if (comm->internode_table) {
+        return comm->internode_table[r];
+    } else {
+        /* canonical or trivial */
+        return r / comm->num_local;
+    }
 }
 
 MPL_STATIC_INLINE_PREFIX int MPIR_Get_intranode_rank(MPIR_Comm * comm, int r)
 {
-    /* TODO: optimize for special ones such as node_consecutive and balanced */
     MPIR_Assert(comm->attr | MPIR_COMM_ATTR__HIERARCHY);
-    MPIR_Assert(comm->intranode_table);
-
-    return comm->intranode_table[r];
+    if (comm->intranode_table) {
+        return comm->intranode_table[r];
+    } else {
+        /* canonical or trivial */
+        if ((comm->rank / comm->num_local) == (r / comm->num_local)) {
+            return r % comm->num_local;
path: src/include/mpir_comm.h
user: login: raffenet
body: is there another minor optimization to be had in the node consecutive case where we can subtract and offset rather than use the modulo operator? otherwise this looks good.
created_at: 2025-04-07T16:23:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -514,6 +514,28 @@ static int check_hierarchy(MPIR_Comm * comm)
     comm->node_comm = NULL;
     comm->node_roots_comm = NULL;
 
+    if (num_nodes == 1) {
+        comm->hierarchy_flags |= MPIR_COMM_HIERARCHY__SINGLE_NODE;
+    } else if (num_local == 1) {
+        comm->hierarchy_flags |= MPIR_COMM_HIERARCHY__NO_LOCAL;
+    } else {
+        bool is_node_consecutive = true;
+        for (int i = 1; i < comm_size; i++) {
+            if (internode_table[i] < internode_table[i - 1]) {
+                is_node_consecutive = false;
+                break;
+            }
+        }
+
+        if (is_node_consecutive) {
+            comm->hierarchy_flags |= MPIR_COMM_HIERARCHY__NODE_CONSECUTIVE;
+        }
+
+        if (num_local * num_nodes == comm_size) {
+            comm->hierarchy_flags |= MPIR_COMM_HIERARCHY__NODE_BALANCED;
+        }
path: src/mpi/comm/commutil.c
user: login: hzhou
body: You are right. The `num_local` is only the size of the local node. Let me update it by iterating through the node map.
created_at: 2025-04-07T16:45:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -121,6 +121,9 @@ enum MPIR_COMM_HINT_PREDEFINED_t {
   S*/
 struct MPIR_Comm {
     MPIR_OBJECT_HEADER;         /* adds handle and ref_count fields */
+    int attr;                   /* if attr is 0, only the core set of fields are set.
path: src/include/mpir_comm.h
user: login: hzhou
body: I am open to suggestions. But I think it is ok. There is no way people will confuse user attributes with this since there is no way user attributes can fit into the simplicity of single `int` :) 
created_at: 2025-04-07T16:47:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -344,20 +344,28 @@ MPL_STATIC_INLINE_PREFIX int MPIR_Stream_comm_set_attr(MPIR_Comm * comm, int src
 
 MPL_STATIC_INLINE_PREFIX int MPIR_Get_internode_rank(MPIR_Comm * comm, int r)
 {
-    /* TODO: optimize for special ones such as node_consecutive and balanced */
     MPIR_Assert(comm->attr | MPIR_COMM_ATTR__HIERARCHY);
-    MPIR_Assert(comm->internode_table);
-
-    return comm->internode_table[r];
+    if (comm->internode_table) {
+        return comm->internode_table[r];
+    } else {
+        /* canonical or trivial */
+        return r / comm->num_local;
+    }
 }
 
 MPL_STATIC_INLINE_PREFIX int MPIR_Get_intranode_rank(MPIR_Comm * comm, int r)
 {
-    /* TODO: optimize for special ones such as node_consecutive and balanced */
     MPIR_Assert(comm->attr | MPIR_COMM_ATTR__HIERARCHY);
-    MPIR_Assert(comm->intranode_table);
-
-    return comm->intranode_table[r];
+    if (comm->intranode_table) {
+        return comm->intranode_table[r];
+    } else {
+        /* canonical or trivial */
+        if ((comm->rank / comm->num_local) == (r / comm->num_local)) {
+            return r % comm->num_local;
path: src/include/mpir_comm.h
user: login: hzhou
body: Yes, but you still need maintain an array of node offsets. I don't think the use case will be common enough for the extra code.
created_at: 2025-04-07T16:50:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -344,20 +344,28 @@ MPL_STATIC_INLINE_PREFIX int MPIR_Stream_comm_set_attr(MPIR_Comm * comm, int src
 
 MPL_STATIC_INLINE_PREFIX int MPIR_Get_internode_rank(MPIR_Comm * comm, int r)
 {
-    /* TODO: optimize for special ones such as node_consecutive and balanced */
     MPIR_Assert(comm->attr | MPIR_COMM_ATTR__HIERARCHY);
-    MPIR_Assert(comm->internode_table);
-
-    return comm->internode_table[r];
+    if (comm->internode_table) {
+        return comm->internode_table[r];
+    } else {
+        /* canonical or trivial */
+        return r / comm->num_local;
+    }
 }
 
 MPL_STATIC_INLINE_PREFIX int MPIR_Get_intranode_rank(MPIR_Comm * comm, int r)
 {
-    /* TODO: optimize for special ones such as node_consecutive and balanced */
     MPIR_Assert(comm->attr | MPIR_COMM_ATTR__HIERARCHY);
-    MPIR_Assert(comm->intranode_table);
-
-    return comm->intranode_table[r];
+    if (comm->intranode_table) {
+        return comm->intranode_table[r];
+    } else {
+        /* canonical or trivial */
+        if ((comm->rank / comm->num_local) == (r / comm->num_local)) {
+            return r % comm->num_local;
path: src/include/mpir_comm.h
user: login: raffenet
body: I'm only thinking of the node consecutive case and where we've already determined they are on the same node. Something like:
```c
/* canonical or trivial */
if ((comm->rank / comm->num_local) == (r / comm->num_local)) {
    if (comm->hierarchy_flags & MPIR_COMM_HIERARCHY__NODE_CONSECUTIVE) {
        return r - comm->local_offset;
    }
    return r % comm->num_local;
} else {
    return -1;
}
```
But we're already 3 branches deep so maybe it's not worth it.
created_at: 2025-04-07T19:05:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -344,20 +344,28 @@ MPL_STATIC_INLINE_PREFIX int MPIR_Stream_comm_set_attr(MPIR_Comm * comm, int src
 
 MPL_STATIC_INLINE_PREFIX int MPIR_Get_internode_rank(MPIR_Comm * comm, int r)
 {
-    /* TODO: optimize for special ones such as node_consecutive and balanced */
     MPIR_Assert(comm->attr | MPIR_COMM_ATTR__HIERARCHY);
-    MPIR_Assert(comm->internode_table);
-
-    return comm->internode_table[r];
+    if (comm->internode_table) {
+        return comm->internode_table[r];
+    } else {
+        /* canonical or trivial */
+        return r / comm->num_local;
+    }
 }
 
 MPL_STATIC_INLINE_PREFIX int MPIR_Get_intranode_rank(MPIR_Comm * comm, int r)
 {
-    /* TODO: optimize for special ones such as node_consecutive and balanced */
     MPIR_Assert(comm->attr | MPIR_COMM_ATTR__HIERARCHY);
-    MPIR_Assert(comm->intranode_table);
-
-    return comm->intranode_table[r];
+    if (comm->intranode_table) {
+        return comm->intranode_table[r];
+    } else {
+        /* canonical or trivial */
+        if ((comm->rank / comm->num_local) == (r / comm->num_local)) {
+            return r % comm->num_local;
path: src/include/mpir_comm.h
user: login: hzhou
body: You can't reliably use `comm->rank / comm->num_local` when `comm->num_local` may not represent node boundary.
created_at: 2025-04-07T19:27:27Z
author_association: CONTRIBUTOR
diff_hunk: @@ -344,20 +344,28 @@ MPL_STATIC_INLINE_PREFIX int MPIR_Stream_comm_set_attr(MPIR_Comm * comm, int src
 
 MPL_STATIC_INLINE_PREFIX int MPIR_Get_internode_rank(MPIR_Comm * comm, int r)
 {
-    /* TODO: optimize for special ones such as node_consecutive and balanced */
     MPIR_Assert(comm->attr | MPIR_COMM_ATTR__HIERARCHY);
-    MPIR_Assert(comm->internode_table);
-
-    return comm->internode_table[r];
+    if (comm->internode_table) {
+        return comm->internode_table[r];
+    } else {
+        /* canonical or trivial */
+        return r / comm->num_local;
+    }
 }
 
 MPL_STATIC_INLINE_PREFIX int MPIR_Get_intranode_rank(MPIR_Comm * comm, int r)
 {
-    /* TODO: optimize for special ones such as node_consecutive and balanced */
     MPIR_Assert(comm->attr | MPIR_COMM_ATTR__HIERARCHY);
-    MPIR_Assert(comm->intranode_table);
-
-    return comm->intranode_table[r];
+    if (comm->intranode_table) {
+        return comm->intranode_table[r];
+    } else {
+        /* canonical or trivial */
+        if ((comm->rank / comm->num_local) == (r / comm->num_local)) {
+            return r % comm->num_local;
path: src/include/mpir_comm.h
user: login: raffenet
body: I'm suggesting a not-yet-defined `comm->local_offset` or `comm->node_offset` which you could compute at creation time. Anyway we don't need to hold up this PR on it.
created_at: 2025-04-07T19:55:41Z
author_association: CONTRIBUTOR

PR comments:
user: login: hzhou
created_at: 2025-04-06T21:31:07Z
body: test:mpich/ch4/most
test:mpich/ch3/most
author_association: CONTRIBUTOR
user: login: hzhou
created_at: 2025-04-07T17:40:10Z
body: test:mpich/ch4/most
test:mpich/ch3/most

I don't think these are related to this PR:
```
rma.02754 - ./rma/linked_list 4
rma.02793 - ./rma/linked_list_bench_lock_excl 4
rma.02794 - ./rma/linked_list_bench_lock_shr 4
rma.02795 - ./rma/linked_list_bench_lock_shr_nocheck 4 
```
but I wonder whether this indicates a regression at some point.
author_association: CONTRIBUTOR