Repository: DataBiosphere/toil

Issue:
s3 support for toil-cwl-runner?
The quickstart guide states:
>When running a CWL workflow on AWS, input files can be provided either on the local file system or in S3 buckets using s3:// URI references. Final output files will be copied to the local file system of the leader node."

However, I consistently get the error at this point in the code:
`"file." % (knownfile, self.pathmapper.mapper(knownfile)[0]))`
reporting:
`WorkflowException: Input file https://s3.amazonaws.com/<bucket>/<file> (at //s3.amazonaws.com/<bucket>/<file>) not found or is not a regular file.`

or, if I use the `s3://` URI system I get:
`WorkflowException: Input file s3://<bucket>/<file> (at //<bucket>/<file>) not found or is not a regular file.`

where `<bucket>` and `<file>` are obviously filled in with my specific bucket and file.

┆Issue is synchronized with this [JIRA Story](https://ucsc-cgl.atlassian.net/browse/TOIL-103)
┆Issue Number: TOIL-103



Discussion:
Not sure what's going on there.  Will try to give it a go this weekend.
Just wondering if there was an update to this? I am also getting this error in Toil version 3.18.1a1
I ended up having to create utilities to transfer files from S3 to the leader node.

Resolving patch:
diff --git a/src/toil/cwl/cwltoil.py b/src/toil/cwl/cwltoil.py
--- a/src/toil/cwl/cwltoil.py
+++ b/src/toil/cwl/cwltoil.py
@@ -305,7 +305,7 @@ def toil_get_file(file_store, index, existing, file_store_id):
     """Get path to input file from Toil jobstore."""
 
     if not file_store_id.startswith("toilfs:"):
-        return schema_salad.ref_resolver.file_uri(file_store_id)
+        return file_store.jobStore.getPublicUrl(file_store.jobStore.importFile(file_store_id))
     src_path = file_store.readGlobalFile(file_store_id[7:])
     index[src_path] = file_store_id
     existing[file_store_id] = src_path
@@ -355,7 +355,7 @@ def uploadFile(uploadfunc, fileindex, existing, uf, skip_broken=False):
         return
     if not uf["location"] and uf["path"]:
         uf["location"] = schema_salad.ref_resolver.file_uri(uf["path"])
-    if not os.path.isfile(uf["location"][7:]):
+    if uf["location"].startswith("file://") and not os.path.isfile(uf["location"][7:]):
         if skip_broken:
             return
         else:


Test patch:
diff --git a/src/toil/test/cwl/cwlTest.py b/src/toil/test/cwl/cwlTest.py
--- a/src/toil/test/cwl/cwlTest.py
+++ b/src/toil/test/cwl/cwlTest.py
@@ -92,6 +92,12 @@ def revsort(self, cwl_filename, tester_fn):
                   'src/toil/test/cwl/revsort-job.json',
                   self._expected_revsort_output(self.outDir))
 
+    def download(self,inputs, tester_fn):
+        input_location = os.path.join('src/toil/test/cwl',inputs)
+        tester_fn('src/toil/test/cwl/download.cwl',
+                  input_location,
+                  self._expected_download_output(self.outDir))
+
     def test_run_revsort(self):
         self.revsort('revsort.cwl', self._tester)
 
@@ -101,6 +107,15 @@ def test_run_revsort2(self):
     def test_run_revsort_debug_worker(self):
         self.revsort('revsort.cwl', self._debug_worker_tester)
 
+    def test_run_s3(self):
+        self.download('download_s3.json', self._tester)
+
+    def test_run_http(self):
+        self.download('download_http.json', self._tester)
+
+    def test_run_https(self):
+        self.download('download_https.json', self._tester)
+
     @slow
     def test_bioconda(self):
         self._tester('src/toil/test/cwl/seqtk_seq.cwl',
@@ -245,3 +260,17 @@ def _expected_revsort_output(outDir):
                 u'size': 1111,
                 u'class': u'File',
                 u'checksum': u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}}
+
+    @staticmethod
+    def _expected_download_output(outDir):
+        # Having unicode string literals isn't necessary for the assertion but
+        # makes for a less noisy diff in case the assertion fails.
+        loc = 'file://' + os.path.join(outDir, 'output.txt')
+        loc = str(loc) if not isinstance(loc, text_type) else loc  # py2/3
+        return {
+            u'output': {
+                u'location': loc,
+                u'basename': u'output.txt',
+                u'size': 0,
+                u'class': u'File',
+                u'checksum': u'sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709'}}
diff --git a/src/toil/test/cwl/download.cwl b/src/toil/test/cwl/download.cwl
new file mode 100644
--- /dev/null
+++ b/src/toil/test/cwl/download.cwl
@@ -0,0 +1,21 @@
+# Example command line program wrapper for the Unix tool "sort"
+# demonstrating command line flags.
+class: CommandLineTool
+doc: "Download a file from different cloud environments"
+cwlVersion: v1.0
+
+
+inputs:
+  - id: input
+    type: File
+    inputBinding:
+      position: 1
+
+outputs:
+  output:
+    type: File
+    outputBinding:
+      glob: output.txt
+
+baseCommand: ["test","-f"]
+stdout: output.txt
\ No newline at end of file
diff --git a/src/toil/test/cwl/download_http.json b/src/toil/test/cwl/download_http.json
new file mode 100644
--- /dev/null
+++ b/src/toil/test/cwl/download_http.json
@@ -0,0 +1,6 @@
+{
+  "input": {
+    "class": "File",
+    "location": "http://toil-datasets.s3.amazonaws.com/wdl_templates.zip"
+  }
+}
\ No newline at end of file
diff --git a/src/toil/test/cwl/download_https.json b/src/toil/test/cwl/download_https.json
new file mode 100644
--- /dev/null
+++ b/src/toil/test/cwl/download_https.json
@@ -0,0 +1,6 @@
+{
+  "input": {
+    "class": "File",
+    "location": "https://toil-datasets.s3.amazonaws.com/wdl_templates.zip"
+  }
+}
\ No newline at end of file
diff --git a/src/toil/test/cwl/download_s3.json b/src/toil/test/cwl/download_s3.json
new file mode 100644
--- /dev/null
+++ b/src/toil/test/cwl/download_s3.json
@@ -0,0 +1,6 @@
+{
+  "input": {
+    "class": "File",
+    "location": "s3://toil-datasets/wdl_templates.zip"
+  }
+}
\ No newline at end of file