Repository: Duke-GCB/bespin-cli

Issue:
Allow using non-DukeDS File parameters
Currently all files are expected to begin with the `dds:` prefix. If a user attempts to use a ` class: File` input specifying either a url or a file path they will receive an error.
```
Invalid DukeDS file path (missing prefix): /data/exome-seq/b37/SUPERSNIPS.vcf
```
If a user attempts to use the location option they receive the following:
```
...
    self.on_class_value(top_level_key, obj)
  File "/Users/jpb67/Documents/work/bespin-cli/env3/lib/python3.6/site-packages/bespin/commands.py", line 424, in on_class_value
    if value['class'] == 'File' and value['path'] in USER_PLACEHOLDER_VALUES:
KeyError: 'path'
```

Documentation on CWL File:
https://www.commonwl.org/v1.0/Workflow.html#File



Discussion:

Patch:
diff --git a/bespin/commands.py b/bespin/commands.py
index e30d252..1766e96 100644
--- a/bespin/commands.py
+++ b/bespin/commands.py
@@ -3,6 +3,7 @@ from bespin.config import ConfigFile
 from bespin.api import BespinApi
 from bespin.exceptions import IncompleteJobFileException
 from bespin.dukeds import DDSFileUtil
+from bespin.dukeds import PATH_PREFIX as DUKEDS_PATH_PREFIX
 from tabulate import tabulate
 import yaml
 import json
@@ -411,8 +412,8 @@ class JobOrderWalker(object):
         :param path: str: format dds://<projectname>/<filepath>
         :return: str: file path to be used for staging data when running the workflow
         """
-        if path.startswith("dds://"):
-            return path.replace("dds://", "dds_").replace("/", "_").replace(":", "_")
+        if path.startswith(DUKEDS_PATH_PREFIX):
+            return path.replace(DUKEDS_PATH_PREFIX, "dds_").replace("/", "_").replace(":", "_")
         return path
 
 
@@ -421,8 +422,10 @@ class JobOrderPlaceholderCheck(JobOrderWalker):
         self.keys_with_placeholders = set()
 
     def on_class_value(self, top_level_key, value):
-        if value['class'] == 'File' and value['path'] in USER_PLACEHOLDER_VALUES:
-            self.keys_with_placeholders.add(top_level_key)
+        if value['class'] == 'File':
+            path = value.get('path')
+            if path and path in USER_PLACEHOLDER_VALUES:
+                self.keys_with_placeholders.add(top_level_key)
 
     def on_simple_value(self, top_level_key, value):
         if value in USER_PLACEHOLDER_VALUES:
@@ -432,7 +435,9 @@ class JobOrderPlaceholderCheck(JobOrderWalker):
 class JobOrderFormatFiles(JobOrderWalker):
     def on_class_value(self, top_level_key, value):
         if value['class'] == 'File':
-            value['path'] = self.format_file_path(value['path'])
+            path = value.get('path')
+            if path:
+                value['path'] = self.format_file_path(path)
 
 
 class JobOrderFileDetails(JobOrderWalker):
@@ -442,6 +447,7 @@ class JobOrderFileDetails(JobOrderWalker):
 
     def on_class_value(self, top_level_key, value):
         if value['class'] == 'File':
-            path = value['path']
-            dds_file = self.dds_file_util.find_file_for_path(path)
-            self.dds_files.append((dds_file, self.format_file_path(path)))
+            path = value.get('path')
+            if path and path.startswith(DUKEDS_PATH_PREFIX):
+                dds_file = self.dds_file_util.find_file_for_path(path)
+                self.dds_files.append((dds_file, self.format_file_path(path)))


Test patch:
diff --git a/bespin/test_commands.py b/bespin/test_commands.py
index 486bf81..c03c245 100644
--- a/bespin/test_commands.py
+++ b/bespin/test_commands.py
@@ -250,6 +250,14 @@ class JobFileTestCase(TestCase):
                 'class': 'File',
                 'path': 'dds://project/somepath.txt'
             },
+            'my_path_file': {
+                'class': 'File',
+                'path': '/tmp/data.txt'
+            },
+            'my_url_file': {
+                'class': 'File',
+                'location': 'https://github.com/datafile1.dat'
+            },
             'myint': 123,
             'myfileary': [
                 {
@@ -298,6 +306,14 @@ class JobFileTestCase(TestCase):
                  'path': 'dds_myproject_rawData_SAAAA_R2_001.fastq.gz'
              },
              'name': 'Sample1'}])
+        self.assertEqual(user_job_order['my_path_file'], {
+            'class': 'File',
+            'path': '/tmp/data.txt'
+        }, "Plain file paths should not be modified.")
+        self.assertEqual(user_job_order['my_url_file'], {
+                'class': 'File',
+                'location': 'https://github.com/datafile1.dat'
+        }, "URL file paths should not be modified.")
 
     @patch('bespin.commands.DDSFileUtil')
     def test_get_dds_files_details(self, mock_dds_file_util):
@@ -529,7 +545,15 @@ class JobOrderWalkerTestCase(TestCase):
                     'class': 'File',
                     'path': 'somepath3'
                 }]
-            }
+            },
+            'plain_path_file': {
+                'class': 'File',
+                'path': '/tmp/data.txt'
+            },
+            'url_file': {
+                'class': 'File',
+                'location': 'https://github.com/datafile1.dat'
+            },
         })
 
         walker.on_simple_value.assert_has_calls([
@@ -592,6 +616,14 @@ class JobOrderPlaceholderCheckTestCase(TestCase):
                     'path': FILE_PLACEHOLDER,
                 }
             },
+            'plain_path_file': {
+                'class': 'File',
+                'path': '/tmp/data.txt'
+            },
+            'url_file': {
+                'class': 'File',
+                'location': 'https://github.com/datafile1.dat'
+            },
         }
         expected_keys = [
             'bad_str', 'bad_int', 'bad_file', 'bad_str_ary', 'bad_file_ary', 'bad_file_dict',
@@ -623,6 +655,14 @@ class JobOrderFormatFilesTestCase(TestCase):
                     'path': 'dds://project3/data/other/somepath.txt',
                 }
             },
+            'plain_path_file': {
+                'class': 'File',
+                'path': '/tmp/data.txt'
+            },
+            'url_file': {
+                'class': 'File',
+                'location': 'https://github.com/datafile1.dat'
+            },
         }
 
         formatter = JobOrderFormatFiles()
@@ -661,6 +701,14 @@ class JobOrderFileDetailsTestCase(TestCase):
                     'path': 'dds://project3/data/other/somepath.txt',
                 }
             },
+            'plain_path_file': {
+                'class': 'File',
+                'path': '/tmp/data.txt'
+            },
+            'url_file': {
+                'class': 'File',
+                'location': 'https://github.com/datafile1.dat'
+            },
         }
         expected_dds_file_info = [
             ('ddsfiledata', 'dds_project1_data_somepath.txt'),