Repository: DataBiosphere/toil

Issue:
Update WDL lexer
The WDL version needs to be updated in order to support the newer standard library functions as discussed in https://github.com/DataBiosphere/toil/pull/3364#discussion_r539558414.

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-762)
┆Issue Number: TOIL-762



Discussion:

Resolving patch:
diff --git a/src/toil/wdl/versions/dev.py b/src/toil/wdl/versions/dev.py
--- a/src/toil/wdl/versions/dev.py
+++ b/src/toil/wdl/versions/dev.py
@@ -13,14 +13,21 @@
 # limitations under the License.
 import logging
 
-from toil.wdl.wdl_analysis import AnalyzeWDL
+from wdlparse.dev.WdlLexer import WdlLexer, FileStream
+from wdlparse.dev.WdlParser import WdlParser, CommonTokenStream
+
+from toil.wdl.versions.v1 import AnalyzeV1WDL, is_context
+from toil.wdl.wdl_types import WDLType
 
 logger = logging.getLogger(__name__)
 
 
-class AnalyzeDevelopmentWDL(AnalyzeWDL):
+class AnalyzeDevelopmentWDL(AnalyzeV1WDL):  # extend from 1.0
     """
-    AnalyzeWDL implementation for the development version.
+    AnalyzeWDL implementation for the development version using ANTLR4.
+
+    See: https://github.com/openwdl/wdl/blob/main/versions/development/SPEC.md
+         https://github.com/openwdl/wdl/blob/main/versions/development/parsers/antlr4/WdlParser.g4
     """
 
     @property
@@ -33,4 +40,68 @@ def analyze(self):
         intermediate data structures: `self.workflows_dictionary` and
         `self.tasks_dictionary`.
         """
-        raise NotImplementedError
+        lexer = WdlLexer(FileStream(self.wdl_file))
+        parser = WdlParser(input=CommonTokenStream(lexer))
+        tree = parser.document()
+        self.visit_document(tree)
+
+    def visit_document(self, ctx: WdlParser.DocumentContext) -> None:
+        """
+        Similar to version 1.0, except the 'workflow' element is included in
+        `ctx.document_element()`.
+        """
+        for element in ctx.document_element():
+            self.visit_document_element(element)
+
+    def visit_document_element(self, ctx: WdlParser.Document_elementContext) -> None:
+        """
+        Similar to version 1.0, except this also contains 'workflow'.
+        """
+        element = ctx.children[0]
+        if is_context(element, 'WorkflowContext'):
+            self.visit_workflow(element)
+        else:
+            # let super take care of the rest.
+            super(AnalyzeDevelopmentWDL, self).visit_document_element(ctx)
+
+    def visit_call(self, ctx: WdlParser.CallContext) -> dict:
+        """
+        Similar to version 1.0, except `ctx.call_afters()` is added.
+        """
+        # TODO: implement call_afters
+        # See: https://github.com/openwdl/wdl/blob/main/versions/development/SPEC.md#call-statement
+        return super(AnalyzeDevelopmentWDL, self).visit_call(ctx)
+
+    def visit_string_expr_part(self, ctx: WdlParser.String_expr_partContext) -> str:
+        """
+        Similar to version 1.0, except `ctx.expression_placeholder_option()`
+        is removed.
+        """
+        # expression placeholder options are removed in development
+        # See: https://github.com/openwdl/wdl/blob/main/versions/development/parsers/antlr4/WdlParser.g4#L55
+
+        return self.visit_expr(ctx.expr())
+
+    def visit_wdl_type(self, ctx: WdlParser.Wdl_typeContext) -> WDLType:
+        """
+        Similar to version 1.0, except Directory type is added.
+        """
+        identifier = ctx.type_base().children[0]
+
+        if identifier == 'Directory':
+            # TODO: implement Directory type
+            raise NotImplementedError('Directory type is not implemented.')
+        else:
+            # let super take care of the rest.
+            return super(AnalyzeDevelopmentWDL, self).visit_wdl_type(ctx)
+
+    def visit_expr_core(self, expr: WdlParser.Expr_coreContext) -> str:
+        """
+        Similar to version 1.0, except struct literal is added.
+        """
+        if is_context(expr, 'Struct_literalContext'):
+            # TODO: implement struct literal
+            raise NotImplementedError(f'WDL struct is not implemented.')
+        else:
+            # let super take care of the rest.
+            return super(AnalyzeDevelopmentWDL, self).visit_expr_core(expr)
diff --git a/src/toil/wdl/versions/v1.py b/src/toil/wdl/versions/v1.py
--- a/src/toil/wdl/versions/v1.py
+++ b/src/toil/wdl/versions/v1.py
@@ -23,13 +23,16 @@
 logger = logging.getLogger(__name__)
 
 
-def is_context(ctx, classname: Union[str, tuple]):
+def is_context(ctx, classname: Union[str, tuple]) -> bool:
     """
     Returns whether an ANTLR4 context object is of the precise type `classname`.
 
-    If a tuple is provided as classname, this returns True if the context object
-    matches one of the string class names.
+    :param ctx: An ANTLR4 context object.
+    :param classname: The class name(s) as a string or a tuple of strings. If a
+                      tuple is provided, this returns True if the context object
+                      matches one of the class names.
     """
+    # we check for `ctx.__class__.__name__` so that it's portable across multiple similar auto-generated parsers.
     if isinstance(classname, str):
         return ctx.__class__.__name__ == classname
     return ctx.__class__.__name__ in classname


Test patch: