Repository: DataBiosphere/toil

Issue:
The cluster scalar uses total memory instead of available memory.
The cluster scaler calculation to determine which machine types to start uses the total memory value. The Mesos slaves will return available memory, though, which might not be sufficient to run the jobs. This leads to the situation where the clusterScalar will not spin up additional machines because it thinks that there are enough, but some jobs will hang forever since the memory offered by Mesos is insufficient. This occurred in issue #2078, which was resolved by lowering memory requirements, but did not fix the underlying issue.

┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-233)
┆friendlyId: TOIL-233



Discussion:

Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -155,6 +155,7 @@ def __init__(self) -> None:
         self.nodeStorage: int = 50
         self.nodeStorageOverrides: List[str] = []
         self.metrics: bool = False
+        self.assume_zero_overhead: bool = False
 
         # Parameters to limit service jobs, so preventing deadlock scheduling scenarios
         self.maxPreemptableServiceJobs: int = sys.maxsize
@@ -346,6 +347,7 @@ def parse_int_list(s: str) -> List[int]:
             raise RuntimeError(f'betaInertia ({self.betaInertia}) must be between 0.0 and 0.9!')
         set_option("scaleInterval", float)
         set_option("metrics")
+        set_option("assume_zero_overhead")
         set_option("preemptableCompensation", float)
         if not 0.0 <= self.preemptableCompensation <= 1.0:
             raise RuntimeError(f'preemptableCompensation ({self.preemptableCompensation}) must be between 0.0 and 1.0!')
@@ -634,6 +636,9 @@ def addOptions(parser: ArgumentParser, config: Optional[Config] = None) -> None:
     autoscaling_options.add_argument("--metrics", dest="metrics", default=False, action="store_true",
                                      help="Enable the prometheus/grafana dashboard for monitoring CPU/RAM usage, "
                                           "queue size, and issued jobs.")
+    autoscaling_options.add_argument("--assumeZeroOverhead", dest="assume_zero_overhead", default=False, action="store_true",
+                                     help="Ignore scheduler and OS overhead and assume jobs can use every last byte "
+                                          "of memory and disk on a node when autoscaling.")
 
     # Parameters to limit service jobs / detect service deadlocks
     if not config.cwl:
diff --git a/src/toil/provisioners/clusterScaler.py b/src/toil/provisioners/clusterScaler.py
--- a/src/toil/provisioners/clusterScaler.py
+++ b/src/toil/provisioners/clusterScaler.py
@@ -11,12 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import copy
 import json
 import logging
+import math
 import os
 import time
 from collections import defaultdict
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple, Union
 
 from toil.batchSystems.abstractBatchSystem import (AbstractBatchSystem,
                                                    AbstractScalableBatchSystem,
@@ -24,9 +26,10 @@
 from toil.bus import ClusterDesiredSizeMessage, ClusterSizeMessage
 from toil.common import Config, defaultTargetTime
 from toil.job import JobDescription, ServiceJobDescription
+from toil.lib.conversions import human2bytes, bytes2human
 from toil.lib.retry import old_retry
 from toil.lib.threading import ExceptionalThread
-from toil.lib.throttle import throttle
+from toil.lib.throttle import LocalThrottle, throttle
 from toil.provisioners.abstractProvisioner import AbstractProvisioner, Shape
 
 if TYPE_CHECKING:
@@ -35,6 +38,19 @@
 
 logger = logging.getLogger(__name__)
 
+# Properties of GKE's memory overhead algorithm
+EVICTION_THRESHOLD = human2bytes('100MiB')
+RESERVE_SMALL_LIMIT = human2bytes('1GiB')
+RESERVE_SMALL_AMOUNT = human2bytes('255MiB')
+RESERVE_BREAKPOINTS: List[Union[int, float]] = [human2bytes('4GiB'), human2bytes('8GiB'), human2bytes('16GiB'), human2bytes('128GiB'), math.inf]
+RESERVE_FRACTIONS = [0.25, 0.2, 0.1, 0.06, 0.02]
+
+# Guess of how much disk space on the root volume is used for the OS and essential container images
+OS_SIZE = human2bytes('5G')
+
+# Define a type for an explanation of why a job can't fit on a node.
+# Consists of a resource name and a constraining value for that resource.
+FailedConstraint = Tuple[str, Union[int, float, bool]]
 
 class BinPackedFit:
     """
@@ -62,8 +78,14 @@ def __init__(self, nodeShapes: List[Shape], targetTime: float = defaultTargetTim
         self.targetTime = targetTime
         self.nodeReservations = {nodeShape: [] for nodeShape in nodeShapes}
 
-    def binPack(self, jobShapes: List[Shape]) -> None:
-        """Pack a list of jobShapes into the fewest nodes reasonable. Can be run multiple times."""
+    def binPack(self, jobShapes: List[Shape]) -> Dict[Shape, List[FailedConstraint]]:
+        """
+        Pack a list of jobShapes into the fewest nodes reasonable.
+        
+        Can be run multiple times.
+        
+        Returns any distinct Shapes that did not fit, mapping to reasons they did not fit.
+        """
         # TODO: Check for redundancy with batchsystems.mesos.JobQueue() sorting
         logger.debug('Running bin packing for node shapes %s and %s job(s).',
                      self.nodeShapes, len(jobShapes))
@@ -72,14 +94,20 @@ def binPack(self, jobShapes: List[Shape]) -> None:
         jobShapes.sort()
         jobShapes.reverse()
         assert len(jobShapes) == 0 or jobShapes[0] >= jobShapes[-1]
+        could_not_fit = {}
         for jS in jobShapes:
-            self.addJobShape(jS)
+            rejection = self.addJobShape(jS)
+            if rejection is not None:
+                # The job bounced because it didn't fit in any bin.
+                could_not_fit[rejection[0]] = rejection[1]
+        return could_not_fit
 
-    def addJobShape(self, jobShape: Shape) -> None:
+    def addJobShape(self, jobShape: Shape) -> Optional[Tuple[Shape, List[FailedConstraint]]]:
         """
-        Add the job to the first node reservation in which it will fit.
-
-        (this is the bin-packing aspect).
+        Add the job to the first node reservation in which it will fit. (This
+        is the bin-packing aspect).
+        
+        Returns the job shape again, and a list of failed constraints, if it did not fit.
         """
         chosenNodeShape = None
         for nodeShape in self.nodeShapes:
@@ -89,16 +117,26 @@ def addJobShape(self, jobShape: Shape) -> None:
                 break
 
         if chosenNodeShape is None:
-            logger.warning("Couldn't fit job with requirements %r into any nodes in the nodeTypes "
-                           "list." % jobShape)
-            return
+            logger.debug("Couldn't fit job with requirements %s into any nodes in the nodeTypes "
+                         "list.", jobShape)
+            # Go back and debug why this happened.
+            fewest_constraints: Optional[List[FailedConstraint]] = None
+            for shape in self.nodeShapes:
+                failures = NodeReservation(nodeShape).get_failed_constraints(jobShape)
+                if fewest_constraints is None or len(failures) < len(fewest_constraints):
+                    # This was closer to fitting.
+                    # TODO: Check the actual constraint values so we don't tell
+                    # the user to raise the memory on the smallest machine?
+                    fewest_constraints = failures
+            
+            return jobShape, fewest_constraints if fewest_constraints is not None else []
 
         # grab current list of job objects appended to this instance type
         nodeReservations = self.nodeReservations[chosenNodeShape]
         for nodeReservation in nodeReservations:
             if nodeReservation.attemptToAddJob(jobShape, chosenNodeShape, self.targetTime):
                 # We succeeded adding the job to this node reservation. Now we're done.
-                return
+                return None
 
         reservation = NodeReservation(chosenNodeShape)
         currentTimeAllocated = chosenNodeShape.wallTime
@@ -111,6 +149,7 @@ def addJobShape(self, jobShape: Shape) -> None:
             currentTimeAllocated += chosenNodeShape.wallTime
             reservation.nReservation = extendThisReservation
             reservation = extendThisReservation
+        return None
 
     def getRequiredNodes(self) -> Dict[Shape, int]:
         """Return a dict from node shape to number of nodes required to run the packed jobs."""
@@ -165,6 +204,29 @@ def __str__(self) -> str:
                 self.nReservation.shape.preemptable if self.nReservation is not None else str(None),
                 str(len(self.shapes())))
 
+    def get_failed_constraints(self, job_shape: Shape) -> List[FailedConstraint]:
+        """
+        Check if a job shape's resource requirements will fit within this allocation.
+        
+        If the job does *not* fit, returns the failing constraints: the resources
+        that can't be accomodated, and the limits that were hit.
+        
+        If the job *does* fit, returns an empty list.
+        
+        Must always agree with fits()! This codepath is slower and used for diagnosis.
+        """
+        
+        failures: List[FailedConstraint] = []
+        if job_shape.memory > self.shape.memory:
+            failures.append(("memory", self.shape.memory))
+        if job_shape.cores > self.shape.cores:
+            failures.append(("cores", self.shape.cores))
+        if job_shape.disk > self.shape.disk:
+            failures.append(("disk", self.shape.disk))
+        if not job_shape.preemptable and self.shape.preemptable:
+            failures.append(("preemptable", self.shape.preemptable))
+        return failures
+    
     def fits(self, jobShape: Shape) -> bool:
         """Check if a job shape's resource requirements will fit within this allocation."""
         return jobShape.memory <= self.shape.memory and \
@@ -290,10 +352,17 @@ def split(
                                   nodeShape.preemptable)))
 
 
-def binPacking(nodeShapes: List[Shape], jobShapes: List[Shape], goalTime: float) -> Dict[Shape, int]:
+def binPacking(nodeShapes: List[Shape], jobShapes: List[Shape], goalTime: float) -> Tuple[Dict[Shape, int], Dict[Shape, List[FailedConstraint]]]:
+    """
+    Using the given node shape bins, pack the given job shapes into nodes to
+    get them done in the given amount of time.
+    
+    Returns a dict saying how many of each node will be needed, a dict from job
+    shapes that could not fit to reasons why.
+    """
     bpf = BinPackedFit(nodeShapes, goalTime)
-    bpf.binPack(jobShapes)
-    return bpf.getRequiredNodes()
+    could_not_fit = bpf.binPack(jobShapes)
+    return bpf.getRequiredNodes(), could_not_fit
 
 
 class ClusterScaler:
@@ -311,6 +380,11 @@ def __init__(
         self.leader = leader
         self.config = config
         self.static: Dict[bool, Dict[str, "Node"]] = {}
+        
+        # If we encounter a Shape of job that we don't think we can run, call
+        # these callbacks with the Shape that didn't fit and the Shapes that
+        # were available.
+        self.on_too_big: List[Callable[[Shape, List[Shape]], Any]] = []
 
         # Dictionary of job names to their average runtime, used to estimate wall time of queued
         # jobs for bin-packing
@@ -370,11 +444,15 @@ def __init__(
 
         self.nodeShapes.sort()
 
+        # Nodes might not actually provide all the resources of their nominal shapes
+        self.node_shapes_after_overhead = self.nodeShapes if config.assume_zero_overhead else [self._reserve_overhead(s) for s in self.nodeShapes]
+        self.without_overhead = {k: v for k, v in zip(self.node_shapes_after_overhead, self.nodeShapes)}
+
         #Node shape to number of currently provisioned nodes
         totalNodes: Dict[Shape, int] = defaultdict(int)
         if isinstance(leader.batchSystem, AbstractScalableBatchSystem) and leader.provisioner:
             for preemptable in (True, False):
-                nodes: List[Node] = []
+                nodes: List["Node"] = []
                 for nodeShape, instance_type in self.nodeShapeToType.items():
                     nodes_thisType = leader.provisioner.getProvisionedWorkers(instance_type=instance_type,
                                                                               preemptable=preemptable)
@@ -416,6 +494,91 @@ def _round(self, number: float) -> int:
             # If we get here, something has gone wrong.
             raise RuntimeError(f"Could not round {number}")
 
+    def _reserve_overhead(self, full_node: Shape) -> Shape:
+        """
+        Given the shape of an entire virtual machine, return the Shape of the
+        resources we expect to actually be available to the backing scheduler
+        (i.e. Mesos or Kubernetes) to run jobs on. Some disk may need to be
+        used to install the OS, and some memory may be needed to run the
+        kernel, Mesos, a Kubelet, Kubernetes system pods, and so on.
+
+        Because shapes are used as dictionary keys, this function must be 1 to
+        1: any two distinct inputs (that are actual node shapes and not
+        pathological inputs off by a single byte) must produce distinct
+        outputs.
+        """
+
+        # See
+        # https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture#memory_cpu
+        # for how EKS works. We are going to just implement their scheme and
+        # hope it is a good enough overestimate of actual overhead.
+        smaller = copy.copy(full_node)
+
+        # Take away some memory to account for how much the scheduler is going to use.
+        smaller.memory -= self._gke_overhead(smaller.memory)
+        # And some disk to account for the OS and possibly superuser reservation
+        # TODO: Figure out if the disk is an OS disk of a scratch disk
+        smaller.disk -= self._disk_overhead(smaller.disk)
+
+        logger.debug('Node shape %s can hold jobs of shape %s', full_node, smaller)
+
+        return smaller
+
+    def _gke_overhead(self, memory_bytes: int) -> int:
+        """
+        Get the number of bytes of memory that Google Kubernetes Engine would
+        reserve or lose to the eviction threshold on a machine with the given
+        number of bytes of memory, according to
+        <https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture#memory_cpu>.
+
+        We assume that this is a good estimate of how much memory will be lost
+        to overhead in a Toil cluster, even though we aren't actually running a
+        GKE cluster.
+        """
+
+        if memory_bytes < RESERVE_SMALL_LIMIT:
+            # On machines with just a little memory, use a flat amount of it.
+            return RESERVE_SMALL_AMOUNT
+
+        # How many bytes are reserved so far?
+        reserved = 0.0
+        # How many bytes of memory have we accounted for so far?
+        accounted: Union[float, int] = 0
+        for breakpoint, fraction in zip(RESERVE_BREAKPOINTS, RESERVE_FRACTIONS):
+            # Below each breakpoint, reserve the matching portion of the memory
+            # since the previous breakpoint, like a progressive income tax.
+            limit = min(breakpoint, memory_bytes)
+            reservation = fraction * (limit - accounted)
+            logger.debug('Reserve %s of memory between %s and %s', bytes2human(reservation), bytes2human(accounted), bytes2human(limit))
+            reserved += reservation
+            accounted = limit
+            if accounted >= memory_bytes:
+                break
+        logger.debug('Reserved %s/%s memory for overhead', bytes2human(reserved), bytes2human(memory_bytes))
+
+        return int(reserved) + EVICTION_THRESHOLD
+
+    def _disk_overhead(self, disk_bytes: int) -> int:
+        """
+        Get the number of bytes of disk that are probably not usable on a
+        machine with a disk of the given size.
+        """
+
+        # How much do we think we need for OS and possible super-user
+        # reservation?
+        disk_needed = OS_SIZE + int(0.05 * disk_bytes)
+
+        if disk_bytes <= disk_needed:
+            # We don't think we can actually use any of this disk
+            logger.warning('All %sB of disk on a node type are likely to be needed by the OS! The node probably cannot do any useful work!', bytes2human(disk_bytes))
+            return disk_bytes
+
+        if disk_needed * 2 > disk_bytes:
+            logger.warning('A node type has only %sB disk, of which more than half are expected to be used by the OS. Consider using a larger --nodeStorage', bytes2human(disk_bytes))
+
+        return disk_needed
+
+
     def getAverageRuntime(self, jobName: str, service: bool = False) -> float:
         if service:
             # We short-circuit service jobs and assume that they will
@@ -497,15 +660,25 @@ def smoothEstimate(self, nodeShape: Shape, estimatedNodeCount: int) -> int:
 
     def getEstimatedNodeCounts(
         self, queuedJobShapes: List[Shape], currentNodeCounts: Dict[Shape, int]
-    ) -> Dict[Shape, int]:
+    ) -> Tuple[Dict[Shape, int], Dict[Shape, List[FailedConstraint]]]:
         """
         Given the resource requirements of queued jobs and the current size of the cluster.
 
-        Returns a dict mapping from nodeShape to the number of nodes we want in the cluster right now.
+        Returns a dict mapping from nodeShape to the number of nodes we want in
+        the cluster right now, and a dict from job shapes that are too big to run
+        on any node to reasons why.
         """
-        nodesToRunQueuedJobs = binPacking(jobShapes=queuedJobShapes,
-                                          nodeShapes=self.nodeShapes,
-                                          goalTime=self.targetTime)
+
+        # Do the bin packing with overhead reserved
+        nodesToRunQueuedJobs, could_not_fit = binPacking(
+            jobShapes=queuedJobShapes,
+            nodeShapes=self.node_shapes_after_overhead,
+            goalTime=self.targetTime
+        )
+        
+        # Then translate back to get results in terms of full nodes without overhead.
+        nodesToRunQueuedJobs = {self.without_overhead[k]: v for k, v in nodesToRunQueuedJobs.items()}
+
         estimatedNodeCounts = {}
         for nodeShape in self.nodeShapes:
             instance_type = self.nodeShapeToType[nodeShape]
@@ -556,7 +729,7 @@ def getEstimatedNodeCounts(
                             self.minNodes[nodeShape])
                 estimatedNodeCount = self.minNodes[nodeShape]
             estimatedNodeCounts[nodeShape] = estimatedNodeCount
-        return estimatedNodeCounts
+        return estimatedNodeCounts, could_not_fit
 
     def updateClusterSize(self, estimatedNodeCounts: Dict[Shape, int]) -> Dict[Shape, int]:
         """
@@ -844,6 +1017,41 @@ def shutDown(self) -> None:
             instance_type = self.nodeShapeToType[nodeShape]
             self.setNodeCount(instance_type=instance_type, numNodes=0, preemptable=preemptable, force=True)
 
+class JobTooBigError(Exception):
+    """
+    Raised in the scaler thread when a job cannot fit in any available node
+    type and is likely to lock up the workflow.
+    """
+    
+    def __init__(self, job: Optional[JobDescription] = None, shape: Optional[Shape] = None, constraints: Optional[List[FailedConstraint]] = None):
+        """
+        Make a JobTooBigError.
+        
+        Can have a job, the job's shape, and the limiting resources and amounts. All are optional.
+        """
+        self.job = job
+        self.shape = shape
+        self.constraints = constraints if constraints is not None else []
+        
+        parts = [
+            f"The job {self.job}" if self.job else "A job",
+            f" with shape {self.shape}" if self.shape else "",
+            " is too big for any available node type."
+        ]
+        
+        if self.constraints:
+            parts.append(" It could have fit if it only needed ")
+            parts.append(", ".join([f"{limit} {resource}" for resource, limit in self.constraints]))
+            parts.append(".") 
+        
+        self.msg = ''.join(parts)
+        super().__init__()
+
+    def __str__(self) -> str:
+        """
+        Stringify the exception, including the message.
+        """
+        return self.msg
 
 class ScalerThread(ExceptionalThread):
     """
@@ -860,12 +1068,15 @@ class ScalerThread(ExceptionalThread):
     is made, else the size of the cluster is adapted. The beta factor is an inertia parameter
     that prevents continual fluctuations in the number of nodes.
     """
-    def __init__(self, provisioner: AbstractProvisioner, leader: "Leader", config: Config) -> None:
+    def __init__(self, provisioner: AbstractProvisioner, leader: "Leader", config: Config, stop_on_exception: bool = False) -> None:
         super().__init__(name='scaler')
         self.scaler = ClusterScaler(provisioner, leader, config)
-
+        
         # Indicates that the scaling thread should shutdown
         self.stop = False
+        # Indicates that we should stop the thread if we encounter an error.
+        # Mostly for testing.
+        self.stop_on_exception = stop_on_exception
 
         self.stats = None
         if config.clusterStats:
@@ -876,7 +1087,7 @@ def __init__(self, provisioner: AbstractProvisioner, leader: "Leader", config: C
             for preemptable in [True, False]:
                 self.stats.startStats(preemptable=preemptable)
             logger.debug("...Cluster stats started.")
-
+            
     def check(self) -> None:
         """
         Attempt to join any existing scaler threads that may have died or finished.
@@ -924,15 +1135,36 @@ def tryRun(self) -> None:
                                 preemptable=nodeShape.preemptable,
                             )
                         )
-                    estimatedNodeCounts = self.scaler.getEstimatedNodeCounts(
+                    estimatedNodeCounts, could_not_fit = self.scaler.getEstimatedNodeCounts(
                         queuedJobShapes, currentNodeCounts
                     )
                     self.scaler.updateClusterSize(estimatedNodeCounts)
                     if self.stats:
                         self.stats.checkStats()
+                    
+                    if len(could_not_fit) != 0: 
+                        # If we have any jobs left over that we couldn't fit, complain.
+                        bad_job: Optional[JobDescription] = None
+                        bad_shape: Optional[Shape] = None
+                        for job, shape in zip(queuedJobs, queuedJobShapes):
+                            # Try and find an example job with an offending shape
+                            if shape in could_not_fit:
+                                bad_job = job
+                                bad_shape = shape
+                                break
+                        if bad_shape is None:
+                            # If we can't find an offending job, grab an arbitrary offending shape.
+                            bad_shape = next(iter(could_not_fit))
+                        
+                        raise JobTooBigError(job=bad_job, shape=bad_shape, constraints=could_not_fit[bad_shape])
+                        
                 except:
-                    logger.exception("Exception encountered in scaler thread. Making a best-effort "
-                                     "attempt to keep going, but things may go wrong from now on.")
+                    if self.stop_on_exception:
+                        logger.critical("Stopping ScalerThread due to an error.")
+                        raise
+                    else:
+                        logger.exception("Exception encountered in scaler thread. Making a best-effort "
+                                         "attempt to keep going, but things may go wrong from now on.")
         self.scaler.shutDown()
 
 class ClusterStats:


Test patch:
diff --git a/src/toil/test/provisioners/clusterScalerTest.py b/src/toil/test/provisioners/clusterScalerTest.py
--- a/src/toil/test/provisioners/clusterScalerTest.py
+++ b/src/toil/test/provisioners/clusterScalerTest.py
@@ -22,7 +22,7 @@
 from contextlib import contextmanager
 from queue import Empty, Queue
 from threading import Event, Thread
-from typing import List, Optional, Set, Tuple
+from typing import Dict, List, Optional, Set, Tuple
 from unittest.mock import MagicMock
 
 from toil.batchSystems.abstractBatchSystem import (AbstractBatchSystem,
@@ -59,6 +59,18 @@
                    cores=32,
                    disk=h2b('600G'),
                    preemptable=False)
+# simplified r5.2xlarge (non-preemptable)
+r5_2xlarge = Shape(wallTime=3600,
+                   memory=h2b('64Gi'),
+                   cores=8,
+                   disk=h2b('50G'),
+                   preemptable=False)
+# simplified r5.4xlarge (non-preemptable)
+r5_4xlarge = Shape(wallTime=3600,
+                   memory=h2b('128Gi'),
+                   cores=16,
+                   disk=h2b('50G'),
+                   preemptable=False)
 # simplified t2.micro (non-preemptable)
 t2_micro = Shape(wallTime=3600,
                  memory=h2b('1G'),
@@ -320,9 +332,10 @@ def testMaxNodes(self):
                                 memory=h2b('1G'),
                                 disk=h2b('2G'),
                                 preemptable=False)] * 1000)
-        estimatedNodeCounts = scaler.getEstimatedNodeCounts(jobShapes, defaultdict(int))
+        estimatedNodeCounts, could_not_fit = scaler.getEstimatedNodeCounts(jobShapes, defaultdict(int))
         self.assertEqual(estimatedNodeCounts[r3_8xlarge], 2)
         self.assertEqual(estimatedNodeCounts[c4_8xlarge_preemptable], 3)
+        self.assertEqual(len(could_not_fit), 0)
 
     def testMinNodes(self):
         """
@@ -332,9 +345,10 @@ def testMinNodes(self):
         self.config.minNodes = [2, 3]
         scaler = ClusterScaler(self.provisioner, self.leader, self.config)
         jobShapes = []
-        estimatedNodeCounts = scaler.getEstimatedNodeCounts(jobShapes, defaultdict(int))
+        estimatedNodeCounts, could_not_fit = scaler.getEstimatedNodeCounts(jobShapes, defaultdict(int))
         self.assertEqual(estimatedNodeCounts[r3_8xlarge], 2)
         self.assertEqual(estimatedNodeCounts[c4_8xlarge_preemptable], 3)
+        self.assertEqual(len(could_not_fit), 0)
 
     def testPreemptableDeficitResponse(self):
         """
@@ -354,7 +368,7 @@ def testPreemptableDeficitResponse(self):
         # preemptableCompensation applies.
         self.config.nodeTypes = [c4_8xlarge_preemptable, c4_8xlarge]
         self.provisioner.setAutoscaledNodeTypes([({t}, None) for t in self.config.nodeTypes])
-        
+
         scaler = ClusterScaler(self.provisioner, self.leader, self.config)
         # Simulate a situation where a previous run caused a
         # "deficit" of 5 preemptable nodes (e.g. a spot bid was lost)
@@ -366,11 +380,12 @@ def testPreemptableDeficitResponse(self):
                            memory=h2b('1G'),
                            disk=h2b('2G'),
                            preemptable=True)] * 1000
-        estimatedNodeCounts = scaler.getEstimatedNodeCounts(jobShapes, defaultdict(int))
+        estimatedNodeCounts, could_not_fit = scaler.getEstimatedNodeCounts(jobShapes, defaultdict(int))
         # We don't care about the estimated size of the preemptable
         # nodes. All we want to know is if we responded to the deficit
         # properly: 0.5 * 5 (preemptableCompensation * the deficit) = 3 (rounded up).
         self.assertEqual(estimatedNodeCounts[self.provisioner.node_shapes_for_testing[1]], 3)
+        self.assertEqual(len(could_not_fit), 0)
 
     def testPreemptableDeficitIsSet(self):
         """
@@ -438,6 +453,122 @@ def testBetaInertia(self):
             scaler.smoothEstimate(c4_8xlarge_preemptable, 100)
         self.assertEqual(scaler.smoothEstimate(c4_8xlarge_preemptable, 100), 100)
 
+    def test_overhead_accounting_large(self):
+        """
+        If a node has a certain raw memory or disk capacity, that won't all be
+        available when it actually comes up; some disk and memory will be used
+        by the OS, and the backing scheduler (Mesos, Kubernetes, etc.).
+
+        Make sure this overhead is accounted for for large nodes.
+        """
+
+        # Use a small node (60G) and a big node (260G)
+
+        # If the job needs 100% of the memory of the instance type, it won't
+        # fit and will need a bigger node.
+        self._check_job_estimate([(c4_8xlarge, 0), (r3_8xlarge, 1)], memory=h2b('60G'))
+
+        # If the job needs 98% of the memory of the instance type, it won't
+        # fit and will need a bigger node.
+        self._check_job_estimate([(c4_8xlarge, 0), (r3_8xlarge, 1)], memory=int(h2b('60G') * 0.98))
+
+        # If the job needs 90% of the memory of the instance type, it will fit.
+        self._check_job_estimate([(c4_8xlarge, 1), (r3_8xlarge, 0)], memory=int(h2b('60G') * 0.90))
+
+        # If the job needs 100% of the disk of the instance type, it won't
+        # fit and will need a bigger node.
+        self._check_job_estimate([(c4_8xlarge, 0), (r3_8xlarge, 1)], disk=h2b('100G'))
+
+        # If the job needs all but 7G of the disk of the instance type, it won't
+        # fit and will need a bigger node.
+        self._check_job_estimate([(c4_8xlarge, 0), (r3_8xlarge, 1)], disk=h2b('93G'))
+
+        # If the job leaves 10% and 10G of the disk free, it fits
+        self._check_job_estimate([(c4_8xlarge, 1), (r3_8xlarge, 0)], disk=h2b('90G'))
+
+    def test_overhead_accounting_small(self):
+        """
+        If a node has a certain raw memory or disk capacity, that won't all be
+        available when it actually comes up; some disk and memory will be used
+        by the OS, and the backing scheduler (Mesos, Kubernetes, etc.).
+
+        Make sure this overhead is accounted for for small nodes.
+        """
+
+        # Use a small node (1G) and a big node (260G)
+
+        # If the job needs 100% of the memory of the instance type, it won't
+        # fit and will need a bigger node.
+        self._check_job_estimate([(t2_micro, 0), (r3_8xlarge, 1)], memory=h2b('1G'))
+
+        # If the job needs all but 100M of the memory of the instance type, it
+        # won't fit and will need a bigger node.
+        self._check_job_estimate([(t2_micro, 0), (r3_8xlarge, 1)], memory=h2b('1G') - h2b('100M'))
+
+        # If the job needs no more than 90% of the memory on the node *and*
+        # leaves at least 384M free for overhead, we can rely on it fitting on a 1G
+        # memory node.
+        jobs = [
+            Shape(
+                wallTime=3600,
+                cores=1,
+                memory=h2b('1G') - h2b('384M'),
+                disk=h2b('2G'),
+                preemptable=True
+            )
+        ]
+        self._check_job_estimate([(t2_micro, 1), (r3_8xlarge, 0)], memory=h2b('1G') - h2b('384M'))
+
+    def test_overhead_accounting_observed(self):
+        """
+        If a node has a certain raw memory or disk capacity, that won't all be
+        available when it actually comes up; some disk and memory will be used
+        by the OS, and the backing scheduler (Mesos, Kubernetes, etc.).
+
+        Make sure this overhead is accounted for so that real-world observed
+        failures cannot happen again.
+        """
+        # In
+        # https://github.com/DataBiosphere/toil/issues/4147#issuecomment-1179587561
+        # a user observed what seems to be a nominally 64 GiB node
+        # (r5.2xlarge?) with Mesos reporting "61.0 GB" available, and a
+        # nominally 32 GiB node with Mesos reporting "29.9 GB" available. It's
+        # not clear if Mesos is thinking in actual GB or GiB here.
+
+        # A 62.5Gi job is sent to the larger node
+        self._check_job_estimate([(r5_2xlarge, 0), (r5_4xlarge, 1)], memory=h2b('62.5 Gi'))
+
+    def _check_job_estimate(self, nodes: List[Tuple[Shape, int]], cores=1, memory=1, disk=1) -> None:
+        """
+        Make sure that a job with the given requirements, when run on the given
+        nodes, produces the given numbers of them.
+        """
+
+        self.provisioner.setAutoscaledNodeTypes([({node}, None) for node, _ in nodes])
+        self.config.targetTime = 1
+        self.config.betaInertia = 0.0
+        # We only need up to one node
+        self.config.maxNodes = [1] * len(nodes)
+        scaler = ClusterScaler(self.provisioner, self.leader, self.config)
+
+        jobs = [
+            Shape(
+                wallTime=3600,
+                cores=cores,
+                memory=memory,
+                disk=disk,
+                preemptable=True
+            )
+        ]
+
+        logger.debug('Try and fit jobs: %s', jobs)
+        counts, could_not_fit = scaler.getEstimatedNodeCounts(jobs, defaultdict(int))
+        for node, count in nodes:
+            seen_count = counts.get(node, 0)
+            if seen_count != count:
+                logger.error('Saw %s/%s instances of node %s', seen_count, count, node)
+            self.assertEqual(seen_count, count)
+        self.assertEqual(len(could_not_fit), 0)
 
 class ScalerThreadTest(ToilTest):
     def _testClusterScaling(self, config, numJobs, numPreemptableJobs, jobShape):
@@ -452,7 +583,7 @@ def _testClusterScaling(self, config, numJobs, numPreemptableJobs, jobShape):
         mock = MockBatchSystemAndProvisioner(config=config, secondsPerJob=2.0)
         mock.setAutoscaledNodeTypes([({t}, None) for t in config.nodeTypes])
         mock.start()
-        clusterScaler = ScalerThread(mock, mock, config)
+        clusterScaler = ScalerThread(mock, mock, config, stop_on_exception=True)
         clusterScaler.start()
         try:
             # Add 100 jobs to complete
@@ -468,9 +599,9 @@ def _testClusterScaling(self, config, numJobs, numPreemptableJobs, jobShape):
                     for _ in range(1000):
                         x = mock.getNodeShape(nodeType=jobShape)
                         iJ = JobDescription(requirements=dict(
-                                                memory=random.choice(list(range(1, x.memory))),
-                                                cores=random.choice(list(range(1, x.cores))),
-                                                disk=random.choice(list(range(1, x.disk))),
+                                                memory=random.randrange(1, x.memory),
+                                                cores=random.randrange(1, x.cores),
+                                                disk=random.randrange(1, x.disk),
                                                 preemptable=preemptable),
                                             jobName='testClusterScaling', unitName='')
                         clusterScaler.addCompletedJob(iJ, random.choice(list(range(1, x.wallTime))))
@@ -509,15 +640,15 @@ def testClusterScaling(self):
         config = Config()
 
         # Make defaults dummy values
-        config.defaultMemory = 1
+        config.defaultMemory = h2b('1Gi')
         config.defaultCores = 1
-        config.defaultDisk = 1
+        config.defaultDisk = h2b('1Gi')
 
         # No preemptable nodes/jobs
         config.maxPreemptableNodes = []  # No preemptable nodes
 
         # Non-preemptable parameters
-        config.nodeTypes = [Shape(20, 10, 10, 10, False)]
+        config.nodeTypes = [Shape(20, h2b('10Gi'), 10, h2b('100Gi'), False)]
         config.minNodes = [0]
         config.maxNodes = [10]
 
@@ -527,23 +658,26 @@ def testClusterScaling(self):
         config.scaleInterval = 3
 
         self._testClusterScaling(config, numJobs=100, numPreemptableJobs=0,
-                                 jobShape=config.nodeTypes[0])
+                                 jobShape=Shape(20, h2b('7Gi'), 10, h2b('80Gi'), False))
 
     @slow
     def testClusterScalingMultipleNodeTypes(self):
 
-        smallNode = Shape(20, 5, 10, 10, False)
-        mediumNode = Shape(20, 10, 10, 10, False)
-        largeNode = Shape(20, 20, 10, 10, False)
+        small_node = Shape(20, h2b('5Gi'), 10, h2b('20Gi'), False)
+        small_job = Shape(20, h2b('3Gi'), 10, h2b('4Gi'), False)
+        medium_node = Shape(20, h2b('10Gi'), 10, h2b('20Gi'), False)
+        medium_job = Shape(20, h2b('7Gi'), 10, h2b('4Gi'), False)
+        large_node = Shape(20, h2b('20Gi'), 10, h2b('20Gi'), False)
+        large_job = Shape(20, h2b('16Gi'), 10, h2b('4Gi'), False)
 
         numJobs = 100
 
         config = Config()
 
         # Make defaults dummy values
-        config.defaultMemory = 1
+        config.defaultMemory = h2b('1Gi')
         config.defaultCores = 1
-        config.defaultDisk = 1
+        config.defaultDisk = h2b('1Gi')
 
         # No preemptable nodes/jobs
         config.preemptableNodeTypes = []
@@ -551,7 +685,7 @@ def testClusterScalingMultipleNodeTypes(self):
         config.maxPreemptableNodes = []  # No preemptable nodes
 
         # Make sure the node types don't have to be ordered
-        config.nodeTypes = [largeNode, smallNode, mediumNode]
+        config.nodeTypes = [large_node, small_node, medium_node]
         config.minNodes = [0, 0, 0]
         config.maxNodes = [10, 10]  # test expansion of this list
 
@@ -562,21 +696,21 @@ def testClusterScalingMultipleNodeTypes(self):
 
         mock = MockBatchSystemAndProvisioner(config=config, secondsPerJob=2.0)
         mock.setAutoscaledNodeTypes([({t}, None) for t in config.nodeTypes])
-        clusterScaler = ScalerThread(mock, mock, config)
+        clusterScaler = ScalerThread(mock, mock, config, stop_on_exception=True)
         clusterScaler.start()
         mock.start()
 
         try:
             # Add small jobs
-            list(map(lambda x: mock.addJob(jobShape=smallNode), list(range(numJobs))))
-            list(map(lambda x: mock.addJob(jobShape=mediumNode), list(range(numJobs))))
+            list(map(lambda x: mock.addJob(jobShape=small_job), list(range(numJobs))))
+            list(map(lambda x: mock.addJob(jobShape=medium_job), list(range(numJobs))))
 
             # Add medium completed jobs
             for i in range(1000):
                 iJ = JobDescription(requirements=dict(
-                                        memory=random.choice(range(smallNode.memory, mediumNode.memory)),
-                                        cores=mediumNode.cores,
-                                        disk=largeNode.cores,
+                                        memory=random.choice(range(small_job.memory, medium_job.memory)),
+                                        cores=medium_job.cores,
+                                        disk=large_job.disk,
                                         preemptable=False),
                                     jobName='testClusterScaling', unitName='')
                 clusterScaler.addCompletedJob(iJ, random.choice(range(1, 10)))
@@ -584,7 +718,7 @@ def testClusterScalingMultipleNodeTypes(self):
             while mock.getNumberOfJobsIssued() > 0 or mock.getNumberOfNodes() > 0:
                 logger.debug("%i nodes currently provisioned" % mock.getNumberOfNodes())
                 # Make sure there are no large nodes
-                self.assertEqual(mock.getNumberOfNodes(nodeType=largeNode), 0)
+                self.assertEqual(mock.getNumberOfNodes(nodeType=large_node), 0)
                 clusterScaler.check()
                 time.sleep(0.5)
         finally:
@@ -593,10 +727,10 @@ def testClusterScalingMultipleNodeTypes(self):
 
         # Make sure jobs ran on both the small and medium node types
         self.assertTrue(mock.totalJobs > 0)
-        self.assertTrue(mock.maxWorkers[smallNode] > 0)
-        self.assertTrue(mock.maxWorkers[mediumNode] > 0)
+        self.assertTrue(mock.maxWorkers[small_node] > 0)
+        self.assertTrue(mock.maxWorkers[medium_node] > 0)
 
-        self.assertEqual(mock.maxWorkers[largeNode], 0)
+        self.assertEqual(mock.maxWorkers[large_node], 0)
 
     @slow
     def testClusterScalingWithPreemptableJobs(self):
@@ -605,16 +739,18 @@ def testClusterScalingWithPreemptableJobs(self):
         """
         config = Config()
 
-        jobShape = Shape(20, 10, 10, 10, False)
-        preemptableJobShape = Shape(20, 10, 10, 10, True)
+        node_shape = Shape(20, h2b('10Gi'), 10, h2b('20Gi'), False)
+        preemptable_node_shape = Shape(20, h2b('10Gi'), 10, h2b('20Gi'), True)
+        job_shape = Shape(20, h2b('7Gi'), 10, h2b('2Gi'), False)
+        preemptable_job_shape = Shape(20, h2b('7Gi'), 10, h2b('2Gi'), True)
 
         # Make defaults dummy values
-        config.defaultMemory = 1
+        config.defaultMemory = h2b('1Gi')
         config.defaultCores = 1
-        config.defaultDisk = 1
+        config.defaultDisk = h2b('1Gi')
 
         # non-preemptable node parameters
-        config.nodeTypes = [jobShape, preemptableJobShape]
+        config.nodeTypes = [node_shape, preemptable_node_shape]
         config.minNodes = [0, 0]
         config.maxNodes = [10, 10]
 
@@ -623,7 +759,7 @@ def testClusterScalingWithPreemptableJobs(self):
         config.betaInertia = 0.9
         config.scaleInterval = 3
 
-        self._testClusterScaling(config, numJobs=100, numPreemptableJobs=100, jobShape=jobShape)
+        self._testClusterScaling(config, numJobs=100, numPreemptableJobs=100, jobShape=job_shape)
 
 
 class MockBatchSystemAndProvisioner(AbstractScalableBatchSystem, AbstractProvisioner):