Repository: DataBiosphere/toil

Issue:
range() implemented for WDL.
Description of WDL function to be added:

```
## Array[Int] range(Int)

Given an integer argument, the `range` function creates an array of integers of length equal to the given argument. For example `range(3)` provides the array: `(0, 1, 2)`.
```
https://github.com/openwdl/wdl/blob/main/versions/development/SPEC.md#arrayint-rangeint

 - [x] Add the following WDL function to https://github.com/DataBiosphere/toil/blob/master/src/toil/wdl/wdl_functions.py
    - Implemented as the built-in, so nothing to add.
 - [ ] Add a test that runs a wdl/json with toil and then checks against cromwell to verify that toil and cromwell have the same behavior
       See: https://github.com/DailyDreaming/wdl-conformance-tests/tree/master/tests/standard_library

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-636)
┆Epic: WDL Standard Library Implementation
┆Issue Number: TOIL-636



Discussion:

Resolving patch:
diff --git a/src/toil/wdl/wdl_analysis.py b/src/toil/wdl/wdl_analysis.py
--- a/src/toil/wdl/wdl_analysis.py
+++ b/src/toil/wdl/wdl_analysis.py
@@ -899,6 +899,9 @@ def parse_declaration_expressn_fncall(self, name, params, es):
                     return es + '_toil_wdl_internal__stdout_file'
                 elif name.source_string == 'stderr':
                     return es + '_toil_wdl_internal__stderr_file'
+                elif name.source_string in ('range', 'zip'):
+                    # replace python built-in functions
+                    es += f'wdl_{name.source_string}('
                 else:
                     es = es + name.source_string + '('
             else:
diff --git a/src/toil/wdl/wdl_functions.py b/src/toil/wdl/wdl_functions.py
--- a/src/toil/wdl/wdl_functions.py
+++ b/src/toil/wdl/wdl_functions.py
@@ -34,6 +34,12 @@
 wdllogger = logging.getLogger(__name__)
 
 
+class WDLRuntimeError(Exception):
+    """ WDL-related run-time error."""
+    def __init__(self, message):
+        super(WDLRuntimeError, self).__init__(message)
+
+
 def glob(glob_pattern, directoryname):
     '''
     Walks through a directory and its subdirectories looking for files matching
@@ -759,7 +765,7 @@ def write_lines(in_lines: List[str],
 
     with open(path, 'w') as file:
         for line in in_lines:
-            file.write(line + '\n')
+            file.write(f'{line}\n')
 
     if file_store:
         file_store.writeGlobalFile(path, cleanup=True)
@@ -839,6 +845,19 @@ def write_map(in_map: Dict[str, str],
     return path
 
 
+def wdl_range(num: int) -> List[int]:
+    """
+    Given an integer argument, the range function creates an array of integers of
+    length equal to the given argument.
+
+    WDL syntax: Array[Int] range(Int)
+    """
+    if not (isinstance(num, int) and num >= 0):
+        raise WDLRuntimeError(f'range() requires an integer greater than or equal to 0 (but got {num})')
+
+    return list(range(num))
+
+
 def transpose(in_array: List[List[Any]]) -> List[List[Any]]:
     """
     Given a two dimensional array argument, the transpose function transposes the
diff --git a/src/toil/wdl/wdl_synthesis.py b/src/toil/wdl/wdl_synthesis.py
--- a/src/toil/wdl/wdl_synthesis.py
+++ b/src/toil/wdl/wdl_synthesis.py
@@ -109,6 +109,7 @@ def write_modules(self):
                     from toil.wdl.wdl_functions import basename
                     from toil.wdl.wdl_functions import floor
                     from toil.wdl.wdl_functions import ceil
+                    from toil.wdl.wdl_functions import wdl_range
                     from toil.wdl.wdl_functions import transpose
                     import fnmatch
                     import textwrap


Test patch:
diff --git a/src/toil/test/wdl/builtinTest.py b/src/toil/test/wdl/builtinTest.py
--- a/src/toil/test/wdl/builtinTest.py
+++ b/src/toil/test/wdl/builtinTest.py
@@ -4,6 +4,8 @@
 import subprocess
 import shutil
 import uuid
+from typing import Optional, List
+
 from toil.wdl.wdl_functions import sub
 from toil.wdl.wdl_functions import ceil
 from toil.wdl.wdl_functions import floor
@@ -27,6 +29,7 @@
 
 class WdlStandardLibraryFunctionsTest(ToilTest):
     """ A set of test cases for toil's wdl functions."""
+
     def setUp(self):
         """Runs anew before each test to create farm fresh temp dirs."""
         self.output_dir = os.path.join('/tmp/', 'toil-wdl-test-' + str(uuid.uuid4()))
@@ -236,22 +239,54 @@ class WdlStandardLibraryWorkflowsTest(ToilTest):
 
     All tests should include a simple wdl and json file for toil to run that checks the output.
     """
+
     @classmethod
     def setUpClass(cls):
         cls.program = os.path.abspath("src/toil/wdl/toilwdl.py")
 
-    def check_function(self, function_name, cases, expected_result):
-        wdl_files = [os.path.abspath(f'src/toil/test/wdl/standard_library/{function_name}_{case}.wdl') for case in cases]
-        json_file = os.path.abspath(f'src/toil/test/wdl/standard_library/{function_name}.json')
+    def check_function(self,
+                       function_name: str,
+                       cases: List[str],
+                       json_file_name: Optional[str] = None,
+                       expected_result: Optional[str] = None,
+                       expected_exception: Optional[str] = None):
+        """
+        Run the given WDL workflow and check its output. The WDL workflow
+        should store its output inside a 'output.txt' file that can be
+        compared to `expected_result`.
+
+        If `expected_exception` is set, this test passes only when both the
+        workflow fails and that the given `expected_exception` string is
+        present in standard error.
+        """
+        wdl_files = [os.path.abspath(f'src/toil/test/wdl/standard_library/{function_name}_{case}.wdl')
+                     for case in cases]
+        json_file = os.path.abspath(f'src/toil/test/wdl/standard_library/{json_file_name or function_name}.json')
         for wdl_file in wdl_files:
             with self.subTest(f'Testing: {wdl_file} {json_file}'):
                 output_dir = f'/tmp/toil-wdl-test-{uuid.uuid4()}'
                 os.makedirs(output_dir)
-                subprocess.check_call([exactPython, self.program, wdl_file, json_file, '-o', output_dir])
-                output = os.path.join(output_dir, 'output.txt')
-                with open(output, 'r') as f:
-                    result = f.read().strip()
-                self.assertEqual(result, expected_result)
+
+                if expected_exception is not None:
+                    with self.assertRaises(subprocess.CalledProcessError) as context:
+                        # use check_output() here so that the output is read before return.
+                        subprocess.check_output([exactPython, self.program, wdl_file, json_file, '-o', output_dir],
+                                                stderr=subprocess.PIPE)
+
+                    stderr = context.exception.stderr
+                    self.assertIsInstance(stderr, bytes)
+                    self.assertIn(expected_exception, stderr.decode('utf-8'))
+
+                elif expected_result is not None:
+                    subprocess.check_call([exactPython, self.program, wdl_file, json_file, '-o', output_dir])
+                    output = os.path.join(output_dir, 'output.txt')
+                    with open(output, 'r') as f:
+                        result = f.read().strip()
+                    self.assertEqual(result, expected_result)
+
+                else:
+                    self.fail("Invalid test. Either `expected_result` or `expected_exception` must be set.")
+
                 shutil.rmtree(output_dir)
 
     def test_sub(self):
@@ -287,7 +322,7 @@ def test_stdout(self):
 
     def test_read(self):
         """ Test the set of WDL read functions."""
-        # NOTE: these tests depends on stdout() and the write_*() functions.
+        # NOTE: these tests depend on stdout() and the write_*() functions.
 
         self.check_function('read_lines', cases=['as_output'],
                             expected_result='line 1\n\t\tline 2 with tabs\n line 3\n\nline 5')
@@ -321,7 +356,22 @@ def test_write(self):
         self.check_function('write_map', cases=['as_command'],
                             expected_result='key1\tvalue1\nkey2\tvalue2')
 
+    def test_range(self):
+        # NOTE: this test depends on write_lines().
+        self.check_function('range', cases=['as_input'],
+                            expected_result='0\n1\n2\n3\n4\n5\n6\n7')
+
+        self.check_function('range', cases=['as_input'],
+                            json_file_name='range_0',
+                            expected_result='')
+
+        self.check_function('range', cases=['as_input'],
+                            json_file_name='range_invalid',
+                            expected_exception='WDLRuntimeError')
+
     def test_transpose(self):
+        # NOTE: this test depends on write_tsv().
+
         # this workflow writes a transposed 2-dimensional array as a TSV file.
         self.check_function('transpose', cases=['as_input'], expected_result='0\t3\n1\t4\n2\t5')
 
diff --git a/src/toil/test/wdl/standard_library/range.json b/src/toil/test/wdl/standard_library/range.json
new file mode 100644
--- /dev/null
+++ b/src/toil/test/wdl/standard_library/range.json
@@ -0,0 +1,3 @@
+{
+  "rangeWorkflow.num": 8
+}
diff --git a/src/toil/test/wdl/standard_library/range_0.json b/src/toil/test/wdl/standard_library/range_0.json
new file mode 100644
--- /dev/null
+++ b/src/toil/test/wdl/standard_library/range_0.json
@@ -0,0 +1,3 @@
+{
+  "rangeWorkflow.num": 0
+}
diff --git a/src/toil/test/wdl/standard_library/range_as_input.wdl b/src/toil/test/wdl/standard_library/range_as_input.wdl
new file mode 100644
--- /dev/null
+++ b/src/toil/test/wdl/standard_library/range_as_input.wdl
@@ -0,0 +1,17 @@
+workflow rangeWorkflow {
+  # this workflow depends on write_lines()
+  Int num
+  call copy_output {input: in_array=range(num)}
+}
+
+task copy_output {
+  Array[Int] in_array
+
+  command {
+    cp ${write_lines(in_array)} output.txt
+  }
+
+  output {
+    File the_output = 'output.txt'
+  }
+}
diff --git a/src/toil/test/wdl/standard_library/range_invalid.json b/src/toil/test/wdl/standard_library/range_invalid.json
new file mode 100644
--- /dev/null
+++ b/src/toil/test/wdl/standard_library/range_invalid.json
@@ -0,0 +1,3 @@
+{
+  "rangeWorkflow.num": -1
+}
diff --git a/src/toil/test/wdl/standard_library/transpose_as_input.wdl b/src/toil/test/wdl/standard_library/transpose_as_input.wdl
--- a/src/toil/test/wdl/standard_library/transpose_as_input.wdl
+++ b/src/toil/test/wdl/standard_library/transpose_as_input.wdl
@@ -1,4 +1,5 @@
 workflow transposeWorkflow {
+  # this workflow depends on write_tsv()
   Array[Array[Int]] in_array
 
   call copy_output {input: in_array=transpose(in_array)}