Repository: DataBiosphere/toil

Issue:
length() implemented as WDL.
Description of WDL function to be added:

```
## Int length(Array[X])

Given an Array, the `length` function returns the number of elements in the Array as an Int.
```
https://github.com/openwdl/wdl/blob/main/versions/development/SPEC.md#int-lengtharrayx

 - [ ] Add the following WDL function to https://github.com/DataBiosphere/toil/blob/master/src/toil/wdl/wdl_functions.py
 - [ ] Add a test that runs a wdl/json with toil and then checks against cromwell to verify that toil and cromwell have the same behavior
       See: https://github.com/DailyDreaming/wdl-conformance-tests/tree/master/tests/standard_library

For example:

```wdl
Array[Int] xs = [ 1, 2, 3 ]
Array[String] ys = [ "a", "b", "c" ]
Array[String] zs = [ ]

Int xlen = length(xs) # 3
Int ylen = length(ys) # 3
Int zlen = length(zs) # 0
```

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-644)
┆Epic: WDL Standard Library Implementation
┆Issue Number: TOIL-644



Discussion:

Resolving patch:
diff --git a/src/toil/wdl/wdl_analysis.py b/src/toil/wdl/wdl_analysis.py
--- a/src/toil/wdl/wdl_analysis.py
+++ b/src/toil/wdl/wdl_analysis.py
@@ -879,9 +879,8 @@ def parse_declaration_expressn_fncall(self, name, params, es):
         """
         Parses out cromwell's built-in function calls.
 
-        Some of these are special
-        and need minor adjustments, for example length(), which is equivalent to
-        python's len() function.
+        Some of these are special and need minor adjustments,
+        for example size() requires a fileStore.
 
         :param name:
         :param params:
@@ -891,10 +890,7 @@ def parse_declaration_expressn_fncall(self, name, params, es):
         # name of the function
         if isinstance(name, wdl_parser.Terminal):
             if name.str:
-                # use python's built-in for length()
-                if name.source_string == 'length':
-                    es = es + 'len('
-                elif name.source_string == 'stdout':
+                if name.source_string == 'stdout':
                     # let the stdout() function reference the generated stdout file path.
                     return es + '_toil_wdl_internal__stdout_file'
                 elif name.source_string == 'stderr':
diff --git a/src/toil/wdl/wdl_functions.py b/src/toil/wdl/wdl_functions.py
--- a/src/toil/wdl/wdl_functions.py
+++ b/src/toil/wdl/wdl_functions.py
@@ -873,3 +873,15 @@ def transpose(in_array: List[List[Any]]) -> List[List[Any]]:
         assert len(arr) == len(in_array[0]), 'transpose() requires all collections have the same size!'
 
     return [list(i) for i in zip(*in_array)]
+
+
+def length(in_array: List[Any]) -> int:
+    """
+    Given an Array, the `length` function returns the number of elements in the Array
+    as an Int.
+    """
+    if not isinstance(in_array, list):
+        # Cromwell throws an exception for anything other than a WDL Array
+        raise WDLRuntimeError(f'length() requires ${in_array} to be a list!  Not: {type(in_array)}')
+
+    return len(in_array)
diff --git a/src/toil/wdl/wdl_synthesis.py b/src/toil/wdl/wdl_synthesis.py
--- a/src/toil/wdl/wdl_synthesis.py
+++ b/src/toil/wdl/wdl_synthesis.py
@@ -111,6 +111,7 @@ def write_modules(self):
                     from toil.wdl.wdl_functions import ceil
                     from toil.wdl.wdl_functions import wdl_range
                     from toil.wdl.wdl_functions import transpose
+                    from toil.wdl.wdl_functions import length
                     import fnmatch
                     import textwrap
                     import subprocess


Test patch:
diff --git a/src/toil/test/wdl/builtinTest.py b/src/toil/test/wdl/builtinTest.py
--- a/src/toil/test/wdl/builtinTest.py
+++ b/src/toil/test/wdl/builtinTest.py
@@ -22,6 +22,7 @@
 from toil.wdl.wdl_functions import write_json
 from toil.wdl.wdl_functions import write_map
 from toil.wdl.wdl_functions import transpose
+from toil.wdl.wdl_functions import length
 
 from toil.version import exactPython
 from toil.test import ToilTest
@@ -230,6 +231,12 @@ def testFn_Transpose(self):
         self.assertEqual([[0]], transpose([[0]]))
         self.assertRaises(AssertionError, transpose, [[0, 1, 2], [3, 4, 5, 6]])
 
+    def testFn_Length(self):
+        """Test the WDL 'length()' built-in."""
+        self.assertEqual(3, length([1, 2, 3]))
+        self.assertEqual(3, length(['a', 'b', 'c']))
+        self.assertEqual(0, length([]))
+
 
 class WdlStandardLibraryWorkflowsTest(ToilTest):
     """
@@ -375,6 +382,17 @@ def test_transpose(self):
         # this workflow writes a transposed 2-dimensional array as a TSV file.
         self.check_function('transpose', cases=['as_input'], expected_result='0\t3\n1\t4\n2\t5')
 
+    def test_length(self):
+        self.check_function('length', cases=['as_input'], expected_result='3')
+
+        self.check_function('length', cases=['as_input'],
+                            json_file_name='length_invalid',
+                            expected_exception='WDLRuntimeError')
+
+        # length() should not work with Map[X, Y].
+        self.check_function('length', cases=['as_input_with_map'],
+                            expected_exception='WDLRuntimeError')
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/src/toil/test/wdl/standard_library/length.json b/src/toil/test/wdl/standard_library/length.json
new file mode 100644
--- /dev/null
+++ b/src/toil/test/wdl/standard_library/length.json
@@ -0,0 +1,7 @@
+{
+  "lengthWorkflow.in_array": [
+    1,
+    2,
+    3
+  ]
+}
diff --git a/src/toil/test/wdl/standard_library/length_as_input.wdl b/src/toil/test/wdl/standard_library/length_as_input.wdl
new file mode 100644
--- /dev/null
+++ b/src/toil/test/wdl/standard_library/length_as_input.wdl
@@ -0,0 +1,16 @@
+workflow lengthWorkflow {
+  Array[Int] in_array
+  call copy_output {input: num=length(in_array)}
+}
+
+task copy_output {
+  Int num
+
+  command {
+    echo ${num} > output.txt
+  }
+
+  output {
+    File the_output = 'output.txt'
+  }
+}
diff --git a/src/toil/test/wdl/standard_library/length_as_input_with_map.json b/src/toil/test/wdl/standard_library/length_as_input_with_map.json
new file mode 100644
--- /dev/null
+++ b/src/toil/test/wdl/standard_library/length_as_input_with_map.json
@@ -0,0 +1,7 @@
+{
+  "lengthWorkflow.in_map": {
+    "1": "a",
+    "2": "b",
+    "3": "c"
+  }
+}
diff --git a/src/toil/test/wdl/standard_library/length_as_input_with_map.wdl b/src/toil/test/wdl/standard_library/length_as_input_with_map.wdl
new file mode 100644
--- /dev/null
+++ b/src/toil/test/wdl/standard_library/length_as_input_with_map.wdl
@@ -0,0 +1,17 @@
+workflow lengthWorkflow {
+  # this workflow should throw an error. length() does not work with Map[X, Y].
+  Map[String, String] in_map
+  call copy_output {input: num=length(in_map)}
+}
+
+task copy_output {
+  Int num
+
+  command {
+    echo ${num} > output.txt
+  }
+
+  output {
+    File the_output = 'output.txt'
+  }
+}
diff --git a/src/toil/test/wdl/standard_library/length_invalid.json b/src/toil/test/wdl/standard_library/length_invalid.json
new file mode 100644
--- /dev/null
+++ b/src/toil/test/wdl/standard_library/length_invalid.json
@@ -0,0 +1,3 @@
+{
+  "lengthWorkflow.in_array": null
+}