Repository: DataBiosphere/toil

Issue:
Weird permissions error when using importFile with file jobStore
I encountered this bug last week, wrote a ticket for it, was about to submit—and then it mysteriously vanished. Well now it's back.

The importFile can change the permissions of the local copy of the file when using the file jobStore This is my best diagnosis although I'm not certain.

When using the file JobStore, [importFile](https://github.com/BD2KGenomics/toil/blob/90bd990f5f0eb2ccc8c2018c244ff43ea2898135/src/toil/jobStores/fileJobStore.py#L165) uses a slight hack/optimization: if the file is being imported from the same disk, instead of copying it, a hard link is made. This shouldn't be a problem except that permissions are stored on the inode.

_THIS_ wouldn't be a problem if not for the fact that somewhere the permissions are changed to read only (I cannot find where in the code).

The code that is causing this error for me is my current version of the [toil-sort-example.py.txt](https://github.com/BD2KGenomics/toil/files/1140489/toil-sort-example.py.txt) (I had to add the .txt so that github would let me upload it) that I was revamping for #1722.

The problem arises from using exportFile because the exporting the file with the same name back to the source directory because we don't have permissions to overwrite it.

It would be awesome if someone else could validate this. by error trace looks like:
>Traceback (most recent call last):
  File "toil-sort-example.py", line 170, in <module>
    main()
  File "toil-sort-example.py", line 165, in main
    toil.exportFile(sorted_file_id, sort_file_url)
  File "/Users/jesse/Documents/genomics/toil/src/toil/common.py", line 900, in exportFile
    self._jobStore.exportFile(jobStoreFileID, dstUrl)
  File "/Users/jesse/Documents/genomics/toil/src/toil/jobStores/abstractJobStore.py", line 327, in exportFile
    self._exportFile(otherCls, jobStoreFileID, dstUrl)
  File "/Users/jesse/Documents/genomics/toil/src/toil/jobStores/fileJobStore.py", line 190, in _exportFile
    shutil.copyfile(self._getAbsPath(jobStoreFileID), self._extractPathFromUrl(url))
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/shutil.py", line 83, in copyfile
    with open(dst, 'wb') as fdst:
IOError: [Errno 13] Permission denied: '/Users/jesse/Documents/genomics/toil/docs/running/file_to_sort.txt'





Discussion:
I can at least point you to where in the code things are changed to be read-only. It's almost certainly in the caching code, either [here](https://github.com/BD2KGenomics/toil/blob/7b1f22ecd3f8aea6d0a86ab56344d7d80bade4ab/src/toil/fileStore.py#L702) or [here](https://github.com/BD2KGenomics/toil/blob/7b1f22ecd3f8aea6d0a86ab56344d7d80bade4ab/src/toil/fileStore.py#L1040).
Bingo! Disabling caching makes everything work as expected! Thanks @joelarmstrong for the leads.

Resolving patch:
diff --git a/src/toil/batchSystems/options.py b/src/toil/batchSystems/options.py
--- a/src/toil/batchSystems/options.py
+++ b/src/toil/batchSystems/options.py
@@ -24,14 +24,20 @@ def _parasolOptions(addOptionFn):
                      "batch is created for jobs with a a unique set of resource requirements. "
                      "default=%i" % 1000)
 
+
 def _singleMachineOptions(addOptionFn):
     addOptionFn("--scale", dest="scale", default=None,
-        help=("A scaling factor to change the value of all submitted tasks's submitted cores. "
-              "Used in singleMachine batch system. default=%s" % 1))
+                help=("A scaling factor to change the value of all submitted tasks's submitted cores. "
+                      "Used in singleMachine batch system. default=%s" % 1))
+    addOptionFn("--linkImports", dest="linkImports", default=False, action='store_true',
+                help=("When using Toil's importFile function for staging, input files are copied to the job store. "
+                      "Specifying this option saves space by hard-linking imported files. As long as caching is "
+                      "enabled Toil will protect the file automatically by changing the permissions to read-only."))
+
 
 def _mesosOptions(addOptionFn):
     addOptionFn("--mesosMaster", dest="mesosMasterAddress", default=None,
-        help=("The host and port of the Mesos master separated by colon. default=%s" % 'localhost:5050'))
+                help=("The host and port of the Mesos master separated by colon. default=%s" % 'localhost:5050'))
 
 # Built in batch systems that have options
 _OPTIONS = [
@@ -42,6 +48,7 @@ def _mesosOptions(addOptionFn):
 
 _options = list(_OPTIONS)
 
+
 def addOptionsDefinition(optionsDefinition):
     _options.append(optionsDefinition)
 
@@ -54,11 +61,11 @@ def setOptions(config, setOption):
 
     batchSystem.setOptions(setOption)
 
-def addOptions(addOptionFn):
 
+def addOptions(addOptionFn):
     addOptionFn("--batchSystem", dest="batchSystem", default=defaultBatchSystem(),
-              help=("The type of batch system to run the job(s) with, currently can be one "
-                    "of %s'. default=%s" % (', '.join(uniqueNames()), defaultBatchSystem())))
+                help=("The type of batch system to run the job(s) with, currently can be one "
+                      "of %s'. default=%s" % (', '.join(uniqueNames()), defaultBatchSystem())))
     addOptionFn("--disableHotDeployment", dest="disableHotDeployment", action='store_true', default=None,
                 help=("Should hot-deployment of the user script be deactivated? If True, the user "
                       "script/package should be present at the same location on all workers. "
@@ -68,17 +75,17 @@ def addOptions(addOptionFn):
         o(addOptionFn)
 
 def setDefaultOptions(config):
-    '''
+    """
     Set default options for builtin batch systems. This is required if a Config
     object is not constructed from an Options object.
-    '''
-
+    """
     config.batchSystem = "singleMachine"
     config.disableHotDeployment = False
     config.environment = {}
 
     # single machine
     config.scale = 1
+    config.linkImports = False
 
     # mesos
     config.mesosMasterAddress = 'localhost:5050'
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -195,6 +195,7 @@ def parseJobStore(s):
         setOption("mesosMasterAddress")
         setOption("parasolCommand")
         setOption("parasolMaxBatches", int, iC(1))
+        setOption("linkImports")
 
         setOption("environment", parseSetEnv)
 
@@ -318,14 +319,16 @@ def _addOptions(addGroupFn, config):
                              "Allows the restart of an existing workflow")
     addOptionFn("--restart", dest="restart", default=None, action="store_true",
                 help="If --restart is specified then will attempt to restart existing workflow "
-                "at the location pointed to by the --jobStore option. Will raise an exception if the workflow does not exist")
+                     "at the location pointed to by the --jobStore option. Will raise an exception "
+                     "if the workflow does not exist")
 
     #
     #Batch system options
     #
 
     addOptionFn = addGroupFn("toil options for specifying the batch system",
-                             "Allows the specification of the batch system, and arguments to the batch system/big batch system (see below).")
+                             "Allows the specification of the batch system, and arguments to the "
+                             "batch system/big batch system (see below).")
     addBatchOptions(addOptionFn)
 
     #
@@ -435,11 +438,10 @@ def _addOptionFn(*name, **kwargs):
     addOptionFn('--defaultPreemptable', dest='defaultPreemptable', action='store_true')
     addOptionFn("--readGlobalFileMutableByDefault", dest="readGlobalFileMutableByDefault",
                 action='store_true', default=None, help='Toil disallows modification of read '
-                                                        'global files by default. This flag makes '
-                                                        'it makes read file mutable by default, '
-                                                        'however it also defeats the purpose of '
-                                                        'shared caching via hard links to save '
-                                                        'space. Default is False')
+                                                        'global files. Setting this flag causes '
+                                                        'them to be mutable. Unfortunately, this '
+                                                        'prevents saving space by caching files '
+                                                        'with hardlinks.')
     addOptionFn('--maxCores', dest='maxCores', default=None, metavar='INT',
                 help='The maximum number of CPU cores to request from the batch system at any one '
                      'time. Standard suffixes like K, Ki, M, Mi, G or Gi are supported. Default '
@@ -468,7 +470,8 @@ def _addOptionFn(*name, **kwargs):
                             "the job may be longer). default=%s" % config.maxJobDuration))
     addOptionFn("--rescueJobsFrequency", dest="rescueJobsFrequency", default=None,
                       help=("Period of time to wait (in seconds) between checking for "
-                            "missing/overlong jobs, that is jobs which get lost by the batch system. Expert parameter. default=%s" % config.rescueJobsFrequency))
+                            "missing/overlong jobs, that is jobs which get lost by the batch "
+                            "system. Expert parameter. default=%s" % config.rescueJobsFrequency))
 
     #
     #Misc options
diff --git a/src/toil/jobStores/fileJobStore.py b/src/toil/jobStores/fileJobStore.py
--- a/src/toil/jobStores/fileJobStore.py
+++ b/src/toil/jobStores/fileJobStore.py
@@ -60,6 +60,7 @@ def __init__(self, path):
         logger.debug("Path to job store directory is '%s'.", self.jobStoreDir)
         # Directory where temporary files go
         self.tempFilesDir = os.path.join(self.jobStoreDir, 'tmp')
+        self.linkImports = None
 
     def initialize(self, config):
         try:
@@ -70,6 +71,7 @@ def initialize(self, config):
             else:
                 raise
         os.mkdir(self.tempFilesDir)
+        self.linkImports = config.linkImports
         super(FileJobStore, self).initialize(config)
 
     def resume(self):
@@ -162,24 +164,32 @@ def jobs(self):
     # Functions that deal with temporary files associated with jobs
     ##########################################
 
+    def _copyOrLink(self, srcURL, destPath):
+        # linking is not done be default because of issue #1755
+        srcPath = self._extractPathFromUrl(srcURL)
+        if self.linkImports:
+            try:
+                os.link(os.path.realpath(srcPath), destPath)
+            except OSError:
+                shutil.copyfile(srcPath, destPath)
+            else:
+                # make imported files read-only if they're linked for protection
+                os.chmod(destPath, 0o444)
+        else:
+            shutil.copyfile(srcPath, destPath)
+
     def _importFile(self, otherCls, url, sharedFileName=None):
         if issubclass(otherCls, FileJobStore):
             if sharedFileName is None:
                 fd, absPath = self._getTempFile()  # use this to get a valid path to write to in job store
                 os.close(fd)
                 os.unlink(absPath)
-                try:
-                    os.link(os.path.realpath(self._extractPathFromUrl(url)), absPath)
-                except OSError:
-                    shutil.copyfile(self._extractPathFromUrl(url), absPath)
+                self._copyOrLink(url, absPath)
                 return FileID(self._getRelativePath(absPath), os.stat(absPath).st_size)
             else:
                 self._requireValidSharedFileName(sharedFileName)
                 path = self._getSharedFilePath(sharedFileName)
-                try:
-                    os.link(os.path.realpath(self._extractPathFromUrl(url)), path)
-                except:
-                    shutil.copyfile(self._extractPathFromUrl(url), path)
+                self._copyOrLink(url, path)
                 return None
         else:
             return super(FileJobStore, self)._importFile(otherCls, url,


Test patch:
diff --git a/src/toil/test/sort/sortTest.py b/src/toil/test/sort/sortTest.py
--- a/src/toil/test/sort/sortTest.py
+++ b/src/toil/test/sort/sortTest.py
@@ -156,7 +156,7 @@ def _toilSort(self, jobStoreLocator, batchSystem,
                 except JobException:
                     pass
                 else:
-                    self.fail('Expected %s to be raised' % JobException )
+                    self.fail('Expected %s to be raised' % JobException)
 
                 # Now check the file is properly sorted..
                 with open(tempSortFile, 'r') as fileHandle:
diff --git a/src/toil/test/src/jobFileStoreTest.py b/src/toil/test/src/jobFileStoreTest.py
--- a/src/toil/test/src/jobFileStoreTest.py
+++ b/src/toil/test/src/jobFileStoreTest.py
@@ -43,6 +43,21 @@ def testNonCachingFileStore(self):
         with Toil(options) as workflow:
             workflow.start(Job.wrapJobFn(simpleFileStoreJob))
 
+    def testImportLinking(self):
+        """
+        importFile will link instead of copying to jobStore in ``--linkImports`` option is specified.
+        we want to test this behavior
+        """
+        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+        options.linkImports = True
+        fileName = 'dummyFile.txt'
+        with open(fileName, 'w') as fh:
+            fh.write('Subtle literature reference.')
+        with Toil(options) as workflow:
+            fileID = workflow.importFile('file://' + os.path.abspath(fileName))
+            workflow.start(Job.wrapJobFn(compareiNodes, fileID, os.path.abspath(fileName)))
+        os.remove(fileName)
+
     def _testJobFileStore(self, retryCount=0, badWorker=0.0, stringNo=1, stringLength=1000000,
                           testNo=2):
         """
@@ -180,3 +195,11 @@ def fileStoreChild(job, testID1, testID2):
 
     for fileID in (testID1, testID2):
         job.fileStore.deleteGlobalFile(fileID)
+
+
+def compareiNodes(job, importedFileID, importedFilePath):
+    localFilePath = os.path.join(job.fileStore.getLocalTempDir(), "childTemp.txt")
+    job.fileStore.readGlobalFile(importedFileID, localFilePath)
+    localiNodeNo = os.stat(localFilePath).st_ino
+    importediNodeNo = os.stat(importedFilePath).st_ino
+    assert localiNodeNo == importediNodeNo