Repository: DataBiosphere/toil

Issue:
using sys.maxsize for options.max{Disk,Memory} hinders 32bit systems
[`sys.maxsize`](https://docs.python.org/3.6/library/sys.html#sys.maxsize) is `2**31 - 1` on 32bit systems, which means that `options.maxMemory` sets a limit of 1.99GiB memory available unless overridden; likewise for `options.maxDisk` being limited to 1.99GiB disk space available unless overridden.

This causes various tests to fail on 32-bit machines


Discussion:

Is there any compelling reason to support 32-bit machines?

size sys.maxsize is not related to available disk, so it seems
to be a bad default.

Michael R. Crusoe <notifications@github.com> writes:
> `sys.maxsize` is `2**31 - 1` on 32bit systems, which means that `options.maxMemory` sets a limit of 1.99GiB memory available unless overridden; likewise for `options.maxDisk` being limited to 1.99GiB disk space available unless overridden.
> 
> This causes various tests to fail on 32-bit machines

> Is there any compelling reason to support 32-bit machines?

If it isn't difficult, I don't see why not. This seems like any easy fix. Perhaps we should use ` math.inf` or `None` to indicate that no value has been provided

Resolving patch:
diff --git a/src/toil/batchSystems/parasol.py b/src/toil/batchSystems/parasol.py
--- a/src/toil/batchSystems/parasol.py
+++ b/src/toil/batchSystems/parasol.py
@@ -26,7 +26,7 @@
 
 from toil.batchSystems.abstractBatchSystem import (BatchSystemSupport,
                                                    UpdatedBatchJobInfo)
-from toil.common import Toil
+from toil.common import SYS_MAX_SIZE, Toil
 from toil.test import get_temp_file
 from toil.lib.iterables import concat
 
@@ -48,7 +48,7 @@ def supportsAutoDeployment(cls):
 
     def __init__(self, config, maxCores, maxMemory, maxDisk):
         super(ParasolBatchSystem, self).__init__(config, maxCores, maxMemory, maxDisk)
-        if maxMemory != sys.maxsize:
+        if maxMemory != SYS_MAX_SIZE:
             logger.warning('The Parasol batch system does not support maxMemory.')
         # Keep the name of the results file for the pstat2 command..
         command = config.parasolCommand
diff --git a/src/toil/batchSystems/singleMachine.py b/src/toil/batchSystems/singleMachine.py
--- a/src/toil/batchSystems/singleMachine.py
+++ b/src/toil/batchSystems/singleMachine.py
@@ -16,7 +16,6 @@
 import os
 import signal
 import subprocess
-import sys
 import time
 import traceback
 from contextlib import contextmanager
@@ -30,7 +29,7 @@
 from toil.batchSystems.abstractBatchSystem import (EXIT_STATUS_UNAVAILABLE_VALUE,
                                                    BatchSystemSupport,
                                                    UpdatedBatchJobInfo)
-from toil.common import Toil
+from toil.common import SYS_MAX_SIZE, Toil
 from toil.lib.threading import cpu_count
 
 log = logging.getLogger(__name__)
@@ -80,12 +79,12 @@ def __init__(self, config, maxCores, maxMemory, maxDisk):
         # If we don't have up to the limit of the resource (and the resource
         # isn't the inlimited sentinel), warn.
         if maxCores > self.numCores:
-            if maxCores != sys.maxsize:
+            if maxCores != SYS_MAX_SIZE:
                 # We have an actually specified limit and not the default
                 log.warning('Not enough cores! User limited to %i but we only have %i.', maxCores, self.numCores)
             maxCores = self.numCores
         if maxMemory > self.physicalMemory:
-            if maxMemory != sys.maxsize:
+            if maxMemory != SYS_MAX_SIZE:
                 # We have an actually specified limit and not the default
                 log.warning('Not enough memory! User limited to %i bytes but we only have %i bytes.', maxMemory, self.physicalMemory)
             maxMemory = self.physicalMemory
@@ -93,7 +92,7 @@ def __init__(self, config, maxCores, maxMemory, maxDisk):
         workdir = Toil.getLocalWorkflowDir(config.workflowID, config.workDir)  # config.workDir may be None; this sets a real directory
         self.physicalDisk = toil.physicalDisk(workdir)
         if maxDisk > self.physicalDisk:
-            if maxDisk != sys.maxsize:
+            if maxDisk != SYS_MAX_SIZE:
                 # We have an actually specified limit and not the default
                 log.warning('Not enough disk space! User limited to %i bytes but we only have %i bytes.', maxDisk, self.physicalDisk)
             maxDisk = self.physicalDisk
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -46,6 +46,9 @@
 
 # aim to pack autoscaling jobs within a 30 minute block before provisioning a new node
 defaultTargetTime = 1800
+SYS_MAX_SIZE = 9223372036854775807
+# sys.max_size on 64 bit systems is 9223372036854775807, so that 32-bit systems
+# use the same number
 UUID_LENGTH = 32
 logger = logging.getLogger(__name__)
 
@@ -104,9 +107,9 @@ def __init__(self):
         self.defaultDisk: int = 2147483648
         self.readGlobalFileMutableByDefault: bool = False
         self.defaultPreemptable: bool = False
-        self.maxCores: int = sys.maxsize
-        self.maxMemory: int = sys.maxsize
-        self.maxDisk: int = sys.maxsize
+        self.maxCores: int = SYS_MAX_SIZE
+        self.maxMemory: int = SYS_MAX_SIZE
+        self.maxDisk: int = SYS_MAX_SIZE
 
         # Retrying/rescuing jobs
         self.retryCount: int = 1
@@ -1393,7 +1396,7 @@ def parseSetEnv(l):
     return d
 
 
-def iC(minValue, maxValue=sys.maxsize):
+def iC(minValue, maxValue=SYS_MAX_SIZE):
     # Returns function that checks if a given int is in the given half-open interval
     assert isinstance(minValue, int) and isinstance(maxValue, int)
     return lambda x: minValue <= x < maxValue


Test patch: