Repository: Libensemble/libensemble

Issue:
Fix LSF resource detection for large jobs on LSF systems (e.g. Summit)
When LSF runs a job, the LSB_HOSTS variable is set to the names of the hosts running the batch job. For a parallel batch job, LSB_HOSTS contains the complete list of hosts that LSF has allocated to that job.

However, If the size of LSB_HOSTS exceeds 4096 bytes, LSB_HOSTS is ignored, and sbatchd instead sets LSB_MCPU_HOSTS which contains the same information in an abbreviated form.

This task is to parse whichever variable supplies the information for nodelist and core counts.

Discussion:

Patch:
diff --git a/libensemble/balsam_controller.py b/libensemble/balsam_controller.py
index 710027e3..a409b2c6 100644
--- a/libensemble/balsam_controller.py
+++ b/libensemble/balsam_controller.py
@@ -141,7 +141,8 @@ class BalsamJobController(MPIJobController):
 
     """
     def __init__(self, auto_resources=True, central_mode=True,
-                 nodelist_env_slurm=None, nodelist_env_cobalt=None):
+                 nodelist_env_slurm=None, nodelist_env_cobalt=None,
+                 nodelist_env_lsf=None, nodelist_env_lsf_shortform=None):
         """Instantiate a new BalsamJobController instance.
 
         A new BalsamJobController object is created with an application
@@ -153,7 +154,10 @@ class BalsamJobController(MPIJobController):
             central_mode = True
 
         super().__init__(auto_resources, central_mode,
-                         nodelist_env_slurm, nodelist_env_cobalt)
+                         nodelist_env_slurm,
+                         nodelist_env_cobalt,
+                         nodelist_env_lsf,
+                         nodelist_env_lsf_shortform)
         self.mpi_launcher = None
         if MPI.COMM_WORLD.Get_rank() == 0:
             BalsamJobController.del_apps()
diff --git a/libensemble/comms/comms.py b/libensemble/comms/comms.py
index 5a992d13..16925e89 100644
--- a/libensemble/comms/comms.py
+++ b/libensemble/comms/comms.py
@@ -125,8 +125,7 @@ class QComm(Comm):
             # QComm._ncomms.value -= 1
 
     def get_num_workers(self):
-        # global _ncomms
-        print('In get function', QComm._ncomms.value)
+        """Return global _ncomms"""
         return QComm._ncomms.value
 
     def send(self, *args):
diff --git a/libensemble/mpi_controller.py b/libensemble/mpi_controller.py
index f140a613..cbd6739a 100644
--- a/libensemble/mpi_controller.py
+++ b/libensemble/mpi_controller.py
@@ -23,7 +23,8 @@ class MPIJobController(JobController):
     def __init__(self, auto_resources=True, central_mode=False,
                  nodelist_env_slurm=None,
                  nodelist_env_cobalt=None,
-                 nodelist_env_lsf=None):
+                 nodelist_env_lsf=None,
+                 nodelist_env_lsf_shortform=None):
         """Instantiate a new JobController instance.
 
         A new JobController object is created with an application
@@ -42,9 +43,9 @@ class MPIJobController(JobController):
 
         central_mode, optional: boolean:
             If true, then running in central mode, else distributed.
-            Central mode means libE processes (manager and workers) are grouped together and
-            do not share nodes with applications. Distributed mode means Workers share nodes
-            with applications.
+            Central mode means libE processes (manager and workers) are
+            grouped together and do not share nodes with applications.
+            Distributed mode means Workers share nodes with applications.
 
         nodelist_env_slurm: String, optional
             The environment variable giving a node list in Slurm format
@@ -62,6 +63,11 @@ class MPIJobController(JobController):
             (Default: Uses LSB_HOSTS) Note: This is only queried
             if a worker_list file is not provided and
             auto_resources=True.
+
+        nodelist_env_lsf_shortform: String, optional
+            The environment variable giving a node list in LSF short-form
+            format (Default: Uses LSB_MCPU_HOSTS) Note: This is only queried
+            if a worker_list file is not provided and auto_resources=True.
         """
 
         JobController.__init__(self)
@@ -91,7 +97,8 @@ class MPIJobController(JobController):
                              launcher=self.mpi_command[0],
                              nodelist_env_slurm=nodelist_env_slurm,
                              nodelist_env_cobalt=nodelist_env_cobalt,
-                             nodelist_env_lsf=nodelist_env_lsf)
+                             nodelist_env_lsf=nodelist_env_lsf,
+                             nodelist_env_lsf_shortform=nodelist_env_lsf_shortform)
 
     def _get_mpi_specs(self, num_procs, num_nodes, ranks_per_node,
                        machinefile, hyperthreads):
@@ -176,7 +183,8 @@ class MPIJobController(JobController):
             runline is printed to logger (At INFO level).
 
         wait_on_run: boolean, optional
-            Whether to wait for job to be polled as RUNNING (or other active/end state) before continuing.
+            Whether to wait for job to be polled as RUNNING (or other
+            active/end state) before continuing.
 
 
         Returns
diff --git a/libensemble/node_resources.py b/libensemble/node_resources.py
index 1d6af6fa..a5fbb25c 100644
--- a/libensemble/node_resources.py
+++ b/libensemble/node_resources.py
@@ -61,12 +61,14 @@ def get_cpu_cores(hyperthreads=False):
 
 
 def _get_local_cpu_resources():
+    """Return logical and physical cores on the local node"""
     logical_cores_avail_per_node = get_cpu_cores(hyperthreads=True)
     physical_cores_avail_per_node = get_cpu_cores(hyperthreads=False)
     return (logical_cores_avail_per_node, physical_cores_avail_per_node)
 
 
 def _print_local_cpu_resources():
+    """Print logical and physical cores on the local node"""
     import sys
     cores_info = _get_local_cpu_resources()
     print(cores_info[0], cores_info[1])
@@ -74,18 +76,29 @@ def _print_local_cpu_resources():
 
 
 def _get_remote_cpu_resources(launcher):
+    """Launch a probe job to obtain logical and physical cores on remote node"""
     import subprocess
     output = subprocess.check_output([launcher, 'python', __file__])
     return output.decode()
 
 
 def _get_cpu_resources_from_env():
-    # May create env resources module to share between other resources modules or send arg.
-    if os.environ['LSB_HOSTS']:
+    """Return logical and physical cores per node by querying environment or None"""
+    found_count = False
+    if 'LSB_HOSTS' in os.environ:
         full_list = os.environ['LSB_HOSTS'].split()
         nodes = [n for n in full_list if 'batch' not in n]
         counter = list(collections.Counter(nodes).values())
-
+        found_count = True
+    elif 'LSB_MCPU_HOSTS' in os.environ:
+        full_list = os.environ['LSB_MCPU_HOSTS'].split()
+        iter_list = iter(full_list)
+        zipped_list = list(zip(iter_list, iter_list))
+        nodes_with_count = [n for n in zipped_list if 'batch' not in n[0]]
+        counter = [int(n[1]) for n in nodes_with_count]
+        found_count = True
+
+    if found_count:
         # Check all nodes have equal cores -  Not doing for other methods currently.
         if len(set(counter)) != 1:
             logger.warning("Detected compute nodes have different core counts: {}".format(set(counter)))
@@ -99,7 +112,7 @@ def _get_cpu_resources_from_env():
 
 
 def get_sub_node_resources(launcher=None, remote_mode=False):
-    """Retruns logical and physical cores per node as a tuple"""
+    """Returns logical and physical cores per node as a tuple"""
     remote_detection = False
     if remote_mode:
         # May be unnecessary condition
diff --git a/libensemble/resources.py b/libensemble/resources.py
index 85b1b146..b45ebc42 100644
--- a/libensemble/resources.py
+++ b/libensemble/resources.py
@@ -54,11 +54,14 @@ class Resources:
     default_nodelist_env_slurm = 'SLURM_NODELIST'
     default_nodelist_env_cobalt = 'COBALT_PARTNAME'
     default_nodelist_env_lsf = 'LSB_HOSTS'
+    default_nodelist_env_lsf_shortform = 'LSB_MCPU_HOSTS'
 
     def __init__(self, top_level_dir=None, central_mode=False, launcher=None,
                  nodelist_env_slurm=None,
                  nodelist_env_cobalt=None,
-                 nodelist_env_lsf=None):
+                 nodelist_env_lsf=None,
+                 nodelist_env_lsf_shortform=None):
+
         """Initialise new Resources instance
 
         Works out the compute resources available for current allocation, including
@@ -93,6 +96,10 @@ class Resources:
             The environment variable giving a node list in LSF format (Default: Uses LSB_HOSTS)
             Note: This is only queried if a worker_list file is not provided and auto_resources=True.
 
+        nodelist_env_lsf_shortform: String, optional
+            The environment variable giving a node list in LSF short-form format (Default: Uses LSB_MCPU_HOSTS)
+            Note: This is only queried if a worker_list file is not provided and auto_resources=True.
+
         """
 
         self.top_level_dir = top_level_dir or os.getcwd()
@@ -100,16 +107,18 @@ class Resources:
         if self.central_mode:
             logger.debug('Running in central mode')
 
-        # These presence of these env vars will be used to detect scheduler
+        # The presence of these env vars will be used to detect scheduler
         self.nodelist_env_slurm = nodelist_env_slurm or Resources.default_nodelist_env_slurm
         self.nodelist_env_cobalt = nodelist_env_cobalt or Resources.default_nodelist_env_cobalt
         self.nodelist_env_lsf = nodelist_env_lsf or Resources.default_nodelist_env_lsf
+        self.nodelist_env_lsf_shortform = nodelist_env_lsf_shortform or Resources.default_nodelist_env_lsf_shortform
 
         # This is global nodelist avail to workers - may change to global_worker_nodelist
         self.global_nodelist = Resources.get_global_nodelist(rundir=self.top_level_dir,
                                                              nodelist_env_slurm=self.nodelist_env_slurm,
                                                              nodelist_env_cobalt=self.nodelist_env_cobalt,
-                                                             nodelist_env_lsf=self.nodelist_env_lsf)
+                                                             nodelist_env_lsf=self.nodelist_env_lsf,
+                                                             nodelist_env_lsf_shortform=self.nodelist_env_lsf_shortform)
 
         remote_detect = False
         self.libE_nodes = Resources.get_libE_nodes()
@@ -237,6 +246,17 @@ class Resources:
         nodes = [n for n in unique_entries if 'batch' not in n]
         return nodes
 
+    @staticmethod
+    def get_lsf_nodelist_frm_shortform(node_list_env):
+        """Get global libEnsemble nodelist from the LSF environment from short-form version"""
+        full_list = os.environ[node_list_env]
+        entries = full_list.split()
+        iter_list = iter(entries)
+        zipped_list = list(zip(iter_list, iter_list))
+        nodes_with_count = [n for n in zipped_list if 'batch' not in n[0]]
+        nodes = [n[0] for n in nodes_with_count]
+        return nodes
+
     # This is for central mode where libE nodes will not share with app nodes
     @staticmethod
     def remove_nodes(global_nodelist_in, remove_list):
@@ -254,7 +274,8 @@ class Resources:
     def get_global_nodelist(rundir=None,
                             nodelist_env_slurm=None,
                             nodelist_env_cobalt=None,
-                            nodelist_env_lsf=None):
+                            nodelist_env_lsf=None,
+                            nodelist_env_lsf_shortform=None):
         """
         Return the list of nodes available to all libEnsemble workers
 
@@ -268,6 +289,7 @@ class Resources:
         nodelist_env_slurm = nodelist_env_slurm or Resources.default_nodelist_env_slurm
         nodelist_env_cobalt = nodelist_env_cobalt or Resources.default_nodelist_env_cobalt
         nodelist_env_lsf = nodelist_env_lsf or Resources.default_nodelist_env_lsf
+        nodelist_env_lsf_shortform = nodelist_env_lsf_shortform or Resources.default_nodelist_env_lsf_shortform
 
         worker_list_file = os.path.join(top_level_dir, 'worker_list')
         global_nodelist = []
@@ -287,6 +309,9 @@ class Resources:
             elif os.environ.get(nodelist_env_lsf):
                 logger.debug("LSF env found - getting nodelist from LSF")
                 global_nodelist = Resources.get_lsf_nodelist(nodelist_env_lsf)
+            elif os.environ.get(nodelist_env_lsf_shortform):
+                logger.debug("LSF env found - getting nodelist from LSF")
+                global_nodelist = Resources.get_lsf_nodelist_frm_shortform(nodelist_env_lsf_shortform)
             else:
                 # Assume a standalone machine if all workers on same node - though give warning.
                 if len(set(Resources.get_libE_nodes())) == 1:


Test patch:
diff --git a/libensemble/tests/unit_tests/test_resources.py b/libensemble/tests/unit_tests/test_resources.py
index 1755a19d..240ca2f1 100644
--- a/libensemble/tests/unit_tests/test_resources.py
+++ b/libensemble/tests/unit_tests/test_resources.py
@@ -151,13 +151,36 @@ def test_lsf_nodelist_seq():
 # def test_lsf_nodelist_reverse_grp():
 
 
+def test_lsf_nodelist_shortform_empty():
+    os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = ""
+    exp_out = []  # empty
+    nodelist = Resources.get_lsf_nodelist_frm_shortform(node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
+    assert nodelist == exp_out, "Nodelist returned does not match expected"
+
+
+def test_lsf_nodelist_shortform_single():
+    os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = 'batch5 1 g06n02 42'
+    exp_out = ["g06n02"]
+    nodelist = Resources.get_lsf_nodelist_frm_shortform(node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
+    assert nodelist == exp_out, "Nodelist returned does not match expected"
+
+
+def test_lsf_nodelist_shortform_seq():
+    os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = 'batch5 1 g06n02 42 h21n18 42'
+    exp_out = ['g06n02', 'h21n18']
+    nodelist = Resources.get_lsf_nodelist_frm_shortform(node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
+    assert nodelist == exp_out, "Nodelist returned does not match expected"
+
+
 # Tests Resources.get_global_nodelist (This requires above tests to work)
 def test_get_global_nodelist_frm_slurm():
     os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "knl-[0020-0022,0137-0139,1234]"
     exp_out = ['knl-0020', 'knl-0021', 'knl-0022', 'knl-0137', 'knl-0138', 'knl-0139', 'knl-1234']
     global_nodelist = Resources.get_global_nodelist(rundir=os.getcwd(),
                                                     nodelist_env_slurm="LIBE_RESOURCES_TEST_NODE_LIST",
-                                                    nodelist_env_cobalt="THIS_ENV_VARIABLE_IS_DEF_NOT_SET")
+                                                    nodelist_env_cobalt="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
+                                                    nodelist_env_lsf="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
+                                                    nodelist_env_lsf_shortform="THIS_ENV_VARIABLE_IS_DEF_NOT_SET")
     assert global_nodelist == exp_out, "global_nodelist returned does not match expected"
 
 
@@ -166,7 +189,9 @@ def test_get_global_nodelist_frm_cobalt():
     exp_out = ['20', '21', '22', '137', '138', '139', '1234']
     global_nodelist = Resources.get_global_nodelist(rundir=os.getcwd(),
                                                     nodelist_env_slurm="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
-                                                    nodelist_env_cobalt="LIBE_RESOURCES_TEST_NODE_LIST")
+                                                    nodelist_env_cobalt="LIBE_RESOURCES_TEST_NODE_LIST",
+                                                    nodelist_env_lsf="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
+                                                    nodelist_env_lsf_shortform="THIS_ENV_VARIABLE_IS_DEF_NOT_SET")
     assert global_nodelist == exp_out, "global_nodelist returned does not match expected"
 
 
@@ -176,7 +201,19 @@ def test_get_global_nodelist_frm_lsf():
     global_nodelist = Resources.get_global_nodelist(rundir=os.getcwd(),
                                                     nodelist_env_slurm="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
                                                     nodelist_env_cobalt="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
-                                                    nodelist_env_lsf="LIBE_RESOURCES_TEST_NODE_LIST")
+                                                    nodelist_env_lsf="LIBE_RESOURCES_TEST_NODE_LIST",
+                                                    nodelist_env_lsf_shortform="THIS_ENV_VARIABLE_IS_DEF_NOT_SET")
+    assert global_nodelist == exp_out, "global_nodelist returned does not match expected"
+
+
+def test_get_global_nodelist_frm_lsf_shortform():
+    os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = 'batch5 1 g06n02 42 h21n18 42'
+    exp_out = ['g06n02', 'h21n18']
+    global_nodelist = Resources.get_global_nodelist(rundir=os.getcwd(),
+                                                    nodelist_env_slurm="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
+                                                    nodelist_env_cobalt="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
+                                                    nodelist_env_lsf="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
+                                                    nodelist_env_lsf_shortform="LIBE_RESOURCES_TEST_NODE_LIST")
     assert global_nodelist == exp_out, "global_nodelist returned does not match expected"
 
 
@@ -185,7 +222,8 @@ def test_get_global_nodelist_standalone():
     global_nodelist = Resources.get_global_nodelist(rundir=os.getcwd(),
                                                     nodelist_env_slurm="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
                                                     nodelist_env_cobalt="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
-                                                    nodelist_env_lsf="THIS_ENV_VARIABLE_IS_DEF_NOT_SET")
+                                                    nodelist_env_lsf="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
+                                                    nodelist_env_lsf_shortform="THIS_ENV_VARIABLE_IS_DEF_NOT_SET")
     assert global_nodelist == [mynode], "global_nodelist returned does not match expected"
 
 
@@ -496,9 +534,14 @@ if __name__ == "__main__":
     test_lsf_nodelist_single()
     test_lsf_nodelist_seq()
 
+    test_lsf_nodelist_shortform_empty()
+    test_lsf_nodelist_shortform_single()
+    test_lsf_nodelist_shortform_seq()
+
     test_get_global_nodelist_frm_slurm()
     test_get_global_nodelist_frm_cobalt()
     test_get_global_nodelist_frm_lsf()
+    test_get_global_nodelist_frm_lsf_shortform()
     test_get_global_nodelist_standalone()
 
     test_get_global_nodelist_frm_wrklst_file()
@@ -510,4 +553,5 @@ if __name__ == "__main__":
     test_get_local_nodelist_distrib_mode_uneven_split()
 
     test_worker_resources()
+
     teardown_standalone_run()