Repository: DataBiosphere/toil

Issue:
Configurable permissions
At the moment, toil sets pretty stringent permissions in the jobstore, e.g. a pipeline can't be restarted by a user different from the original user. In shared HPC environments is plausible to have multiple engineers running the operations. Would be nice if some how one could set default permissions used for files created within the jobstore. Even better if one could tell toil to honor the current `umask` configuration.

┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-273)
┆Issue Number: TOIL-273



Discussion:
We have also experienced a similar issue. We run an automated production which includes an automated reset function. To do this we need to be able to remove the toil directories from a different user on the system. All users are part of the shown group ID 'metagen'. We source two environments: our pipeline environment which specifies umask 002 followed by the toil environment. Toil doesn't appear to honour the initial umask, particularly for folders 'files' and 'jobs' within the job store. We are running with toil version 4.2.0a1-3f4e790c1fa32a2c2257bd5cd203a3b51d5d5661. Please see an example of the permissions below in the path job-store/run/files/no-job

```drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-qf5wej65
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-r00h_tz_
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-rh5keps1
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-s8ifuzti
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-soh8v4wb
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-t75mifhp
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-u78h8x62
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-v74zobuu
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-w4z7pjme
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-xdloi1ph
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-y7f1nik5
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-yct076hl
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-yd59tzgd
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-yt2vbukl
drwx--S---+ 2 emgpr_amplicon metagen 4.0K Jun 17 01:03 file-zewhk8p_

I'm trying to work on this issue. Can anyone share some thoughts on how to approach to this?


Resolving patch:
diff --git a/src/toil/cwl/cwltoil.py b/src/toil/cwl/cwltoil.py
--- a/src/toil/cwl/cwltoil.py
+++ b/src/toil/cwl/cwltoil.py
@@ -769,12 +769,12 @@ def _collectDirEntries(obj: Union[Dict[Text, Any], List[Dict[Text, Any]]]) -> It
                 file_store.exportFile(FileID.unpack(local_file_path), destUrl)
             else:
                 if not os.path.exists(p.target) and p.type == "Directory":
-                    os.makedirs(p.target, 0o0755)
+                    os.makedirs(p.target)
                 if not os.path.exists(p.target) and p.type == "File":
-                    os.makedirs(os.path.dirname(p.target), 0o0755, exist_ok=True)
+                    os.makedirs(os.path.dirname(p.target), exist_ok=True)
                     file_store.exportFile(FileID.unpack(p.resolved[7:]), "file://" + p.target)
                 if not os.path.exists(p.target) and p.type == "CreateFile":
-                    os.makedirs(os.path.dirname(p.target), 0o0755, exist_ok=True)
+                    os.makedirs(os.path.dirname(p.target), exist_ok=True)
                     with open(p.target, "wb") as n:
                         n.write(p.resolved.encode("utf-8"))
 
diff --git a/src/toil/deferred.py b/src/toil/deferred.py
--- a/src/toil/deferred.py
+++ b/src/toil/deferred.py
@@ -36,6 +36,7 @@
 
 class DeferredFunction(namedtuple('DeferredFunction', 'function args kwargs name module')):
     """
+    >>> from collections import defaultdict
     >>> df = DeferredFunction.create(defaultdict, None, {'x':1}, y=2)
     >>> df
     DeferredFunction(defaultdict, ...)
diff --git a/src/toil/jobStores/fileJobStore.py b/src/toil/jobStores/fileJobStore.py
--- a/src/toil/jobStores/fileJobStore.py
+++ b/src/toil/jobStores/fileJobStore.py
@@ -27,6 +27,7 @@
 import stat
 import errno
 import time
+import uuid
 try:
     import cPickle as pickle
 except ImportError:
@@ -565,11 +566,11 @@ def readSharedFileStream(self, sharedFileName):
 
     def writeStatsAndLogging(self, statsAndLoggingString):
         # Temporary files are placed in the stats directory tree
-        fd, tempStatsFile = tempfile.mkstemp(prefix="stats", suffix=".new", dir=self._getArbitraryStatsDir())
+        tempStatsFileName = "stats" + str(uuid.uuid4().hex) + ".new"
+        tempStatsFile = os.path.join(self._getArbitraryStatsDir(), tempStatsFileName)
         writeFormat = 'w' if isinstance(statsAndLoggingString, str) else 'wb'
         with open(tempStatsFile, writeFormat) as f:
             f.write(statsAndLoggingString)
-        os.close(fd)
         os.rename(tempStatsFile, tempStatsFile[:-4])  # This operation is atomic
 
     def readStatsAndLogging(self, callback, readAll=False):
@@ -904,7 +905,11 @@ def _getFileDirectory(self, jobStoreID=None, cleanup=False):
             os.makedirs(jobFilesDir, exist_ok=True)
 
             # Then make a temp directory inside it
-            return tempfile.mkdtemp(prefix='file-', dir=jobFilesDir)
+            filesDir = os.path.join(jobFilesDir, 'file-' + uuid.uuid4().hex)
+            os.mkdir(filesDir)
+            return filesDir
         else:
             # Make a temporary file within the non-job-associated files hierarchy
-            return tempfile.mkdtemp(prefix='file-', dir=self._getArbitraryFilesDir())
+            filesDir = os.path.join(self._getArbitraryFilesDir(), 'file-' + uuid.uuid4().hex)
+            os.mkdir(filesDir)
+            return filesDir


Test patch: