Repository: DataBiosphere/toil

Issue:
os.sysconf('SC_PHYS_PAGES') is throwing an exception
I just rebased my code on top of the latest master branch today... and I'm getting this error:

```
Traceback (most recent call last):
  File "wdltoil.py", line 1, in <module>
    from toil.job import Job
  File "/Users/sfrazer/projects/toil/src/toil/job.py", line 36, in <module>
    from toil.common import loadJobStore
  File "/Users/sfrazer/projects/toil/src/toil/common.py", line 27, in <module>
    from toil.batchSystems.singleMachine import SingleMachineBatchSystem
  File "/Users/sfrazer/projects/toil/src/toil/batchSystems/singleMachine.py", line 31, in <module>
    class SingleMachineBatchSystem(AbstractBatchSystem):
  File "/Users/sfrazer/projects/toil/src/toil/batchSystems/singleMachine.py", line 47, in SingleMachineBatchSystem
    physicalMemory = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
ValueError: unrecognized configuration name
```

It seems that for my machine SC_PHYS_PAGES is not defined:

```
Python 2.7.10 (default, Jul 13 2015, 12:05:58)
[GCC 4.2.1 Compatible Apple LLVM 6.1.0 (clang-602.0.53)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import os
>>> os.sysconf('SC_PAGE_SIZE')
4096
>>> os.sysconf('SC_PHYS_PAGES')
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
ValueError: unrecognized configuration name
```

Any ideas about what I should do?

os.sysconf('SC_PHYS_PAGES') is throwing an exception
I just rebased my code on top of the latest master branch today... and I'm getting this error:

```
Traceback (most recent call last):
  File "wdltoil.py", line 1, in <module>
    from toil.job import Job
  File "/Users/sfrazer/projects/toil/src/toil/job.py", line 36, in <module>
    from toil.common import loadJobStore
  File "/Users/sfrazer/projects/toil/src/toil/common.py", line 27, in <module>
    from toil.batchSystems.singleMachine import SingleMachineBatchSystem
  File "/Users/sfrazer/projects/toil/src/toil/batchSystems/singleMachine.py", line 31, in <module>
    class SingleMachineBatchSystem(AbstractBatchSystem):
  File "/Users/sfrazer/projects/toil/src/toil/batchSystems/singleMachine.py", line 47, in SingleMachineBatchSystem
    physicalMemory = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
ValueError: unrecognized configuration name
```

It seems that for my machine SC_PHYS_PAGES is not defined:

```
Python 2.7.10 (default, Jul 13 2015, 12:05:58)
[GCC 4.2.1 Compatible Apple LLVM 6.1.0 (clang-602.0.53)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import os
>>> os.sysconf('SC_PAGE_SIZE')
4096
>>> os.sysconf('SC_PHYS_PAGES')
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
ValueError: unrecognized configuration name
```

Any ideas about what I should do?



Discussion:
I saw the same error yesterday on my Macbook. I believe that variable isn't
set on osX and we should be using sysctl('HW_REALMEM') instead.. However
python os doesn't have a module for that. Ill get  ask to you if I find a
workaround.

Arjun
On Nov 30, 2015 8:18 AM, "Scott Frazer" notifications@github.com wrote:

> I just rebased my code on top of the latest master branch today... and I'm
> getting this error:
> 
> Traceback (most recent call last):
>   File "wdltoil.py", line 1, in <module>
>     from toil.job import Job
>   File "/Users/sfrazer/projects/toil/src/toil/job.py", line 36, in <module>
>     from toil.common import loadJobStore
>   File "/Users/sfrazer/projects/toil/src/toil/common.py", line 27, in <module>
>     from toil.batchSystems.singleMachine import SingleMachineBatchSystem
>   File "/Users/sfrazer/projects/toil/src/toil/batchSystems/singleMachine.py", line 31, in <module>
>     class SingleMachineBatchSystem(AbstractBatchSystem):
>   File "/Users/sfrazer/projects/toil/src/toil/batchSystems/singleMachine.py", line 47, in SingleMachineBatchSystem
>     physicalMemory = os.sysconf('SC_PAGE_SIZE') \* os.sysconf('SC_PHYS_PAGES')
> ValueError: unrecognized configuration name
> 
> It seems that for my machine SC_PHYS_PAGES is not defined:
> 
> Python 2.7.10 (default, Jul 13 2015, 12:05:58)
> [GCC 4.2.1 Compatible Apple LLVM 6.1.0 (clang-602.0.53)] on darwin
> Type "help", "copyright", "credits" or "license" for more information.
> 
> > > > import os
> > > > os.sysconf('SC_PAGE_SIZE')
> > > > 4096
> > > > os.sysconf('SC_PHYS_PAGES')
> > > > Traceback (most recent call last):
> > > >   File "<stdin>", line 1, in <module>
> > > > ValueError: unrecognized configuration name
> 
> Any ideas about what I should do?
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/BD2KGenomics/toil/issues/587.

Thanks @arkal, for now I just hard coded a value for `os.sysconf('SC_PHYS_PAGES')`

This is a good reference. What value did you give SC_PHYS_PAGES?

http://nadeausoftware.com/articles/2012/09/c_c_tip_how_get_physical_memory_size_system

I hard-coded a value of 600,000 for SC_PHYS_PAGES.  A stop-gap, for sure, but it works

I saw the same error yesterday on my Macbook. I believe that variable isn't
set on osX and we should be using sysctl('HW_REALMEM') instead.. However
python os doesn't have a module for that. Ill get  ask to you if I find a
workaround.

Arjun
On Nov 30, 2015 8:18 AM, "Scott Frazer" notifications@github.com wrote:

> I just rebased my code on top of the latest master branch today... and I'm
> getting this error:
> 
> Traceback (most recent call last):
>   File "wdltoil.py", line 1, in <module>
>     from toil.job import Job
>   File "/Users/sfrazer/projects/toil/src/toil/job.py", line 36, in <module>
>     from toil.common import loadJobStore
>   File "/Users/sfrazer/projects/toil/src/toil/common.py", line 27, in <module>
>     from toil.batchSystems.singleMachine import SingleMachineBatchSystem
>   File "/Users/sfrazer/projects/toil/src/toil/batchSystems/singleMachine.py", line 31, in <module>
>     class SingleMachineBatchSystem(AbstractBatchSystem):
>   File "/Users/sfrazer/projects/toil/src/toil/batchSystems/singleMachine.py", line 47, in SingleMachineBatchSystem
>     physicalMemory = os.sysconf('SC_PAGE_SIZE') \* os.sysconf('SC_PHYS_PAGES')
> ValueError: unrecognized configuration name
> 
> It seems that for my machine SC_PHYS_PAGES is not defined:
> 
> Python 2.7.10 (default, Jul 13 2015, 12:05:58)
> [GCC 4.2.1 Compatible Apple LLVM 6.1.0 (clang-602.0.53)] on darwin
> Type "help", "copyright", "credits" or "license" for more information.
> 
> > > > import os
> > > > os.sysconf('SC_PAGE_SIZE')
> > > > 4096
> > > > os.sysconf('SC_PHYS_PAGES')
> > > > Traceback (most recent call last):
> > > >   File "<stdin>", line 1, in <module>
> > > > ValueError: unrecognized configuration name
> 
> Any ideas about what I should do?
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/BD2KGenomics/toil/issues/587.

Thanks @arkal, for now I just hard coded a value for `os.sysconf('SC_PHYS_PAGES')`

This is a good reference. What value did you give SC_PHYS_PAGES?

http://nadeausoftware.com/articles/2012/09/c_c_tip_how_get_physical_memory_size_system

I hard-coded a value of 600,000 for SC_PHYS_PAGES.  A stop-gap, for sure, but it works


Resolving patch:
diff --git a/src/toil/__init__.py b/src/toil/__init__.py
--- a/src/toil/__init__.py
+++ b/src/toil/__init__.py
@@ -17,6 +17,10 @@
 import os
 import sys
 
+from subprocess import check_output
+
+from bd2k.util import memoize
+
 
 def toilPackageDirPath():
     """
@@ -47,3 +51,18 @@ def resolveEntryPoint(entryPoint):
         # correct version of Toil. This is still better than an absolute path because it gives
         # the user control over Toil's location on both leader and workers.
         return entryPoint
+
+
+@memoize
+def physicalMemory():
+    """
+    >>> n = physicalMemory()
+    >>> n > 0
+    True
+    >>> n == physicalMemory()
+    True
+    """
+    try:
+        return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
+    except ValueError:
+        return int(check_output(['sysctl', '-n', 'hw.memsize']).strip())
diff --git a/src/toil/batchSystems/parasolTestSupport.py b/src/toil/batchSystems/parasolTestSupport.py
--- a/src/toil/batchSystems/parasolTestSupport.py
+++ b/src/toil/batchSystems/parasolTestSupport.py
@@ -9,6 +9,8 @@
 from bd2k.util.files import rm_f
 from bd2k.util.objects import InnerClass
 
+from toil import physicalMemory
+
 log = logging.getLogger(__name__)
 
 
@@ -21,7 +23,7 @@ def _startParasol(self, numCores=None, memory=None):
         if numCores is None:
             numCores = multiprocessing.cpu_count()
         if memory is None:
-            memory = self._physicalMemory()
+            memory = physicalMemory()
         self.numCores = numCores
         self.memory = memory
         self.leader = self.ParasolLeaderThread()
@@ -40,10 +42,6 @@ def _stopParasol(self):
         for path in ('para.results', 'parasol.jid'):
             rm_f(path)
 
-    @classmethod
-    def _physicalMemory(cls):
-        return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
-
     class ParasolThread(threading.Thread):
 
         # Lock is used because subprocess is NOT thread safe: http://tinyurl.com/pkp5pgq
diff --git a/src/toil/batchSystems/singleMachine.py b/src/toil/batchSystems/singleMachine.py
--- a/src/toil/batchSystems/singleMachine.py
+++ b/src/toil/batchSystems/singleMachine.py
@@ -23,6 +23,8 @@
 from threading import Thread
 from threading import Lock, Condition
 from Queue import Queue, Empty
+
+import toil
 from toil.batchSystems.abstractBatchSystem import AbstractBatchSystem
 
 log = logging.getLogger(__name__)
@@ -44,7 +46,7 @@ class SingleMachineBatchSystem(AbstractBatchSystem):
     concurrently.
     """
 
-    physicalMemory = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
+    physicalMemory = toil.physicalMemory()
 
     def __init__(self, config, maxCores, maxMemory, maxDisk):
         if maxCores > self.numCores:


Test patch: