Repository: martinohmann/hcl-rs (rust)

Issue:
Nested function calls leads to extreme parsing times
Here's a heavily reduced version of a terraform variable definition:

```
variable "test" {
  level1 = map(object({
    level2 = map(object({
      level3 = map(object({
        level4 = map(object({
        }))
      }))
    }))
  }))
}
```

On my system the above takes ~24 seconds to parse.

Removing a level by commenting it out:

```
variable "test" {
  level1 = map(object({
    level2 = map(object({
      level3 = map(object({
        # level4 = map(object({
        # }))
      }))
    }))
  }))
}
```

Takes about ~1.77 seconds to parse

Removing one additional level (`level3`) and parsing this file drops to ~0.455 seconds to parse.

It seems that there is something that goes quadratic when having multiple function calls nested. We have a bunch of terraform that heavily uses this to make sure the type of the variable is correct:

```
variable "network_integration" {
  description = "Map of networking integrations between accounts"
  type = map(object({
    friendly_name = string,
    vpcs = map(object({
      id           = string
      cidr         = string
      region       = string
      description  = string
      subnets      = map(string)
      route_tables = map(string)
      security_groups = map(object({
        id = string
        rules = map(object({
          direction   = string
          protocol    = string
          from_port   = string
          to_port     = string
          description = string
        }))
      }))
    }))
    additional_propagated_vpcs   = list(string)
    additional_static_vpc_routes = list(string)
  }))
  default = {}
}
```

As a full example (which this takes ~130 seconds to parse).

Pull request description:
fix(parser): greatly improve expression parsing performance
Previously, the parser needed to do a lot of backtracking while parsing conditionals and operations which massively degraded performance for deeply nested expressions.

This fix inlines the handling for operations and conditionals into the `Expression` rule so that an already matched `ExprTerm` rule does not need to be re-evaluated when no operation/conditional is matched (which is the case in the vast majority of cases since literal values and collections are way more common than these.

Fixes #82 

Patch:
diff --git a/benches/benchmark.rs b/benches/benchmark.rs
index ba7e02a..adfa7d8 100644
--- a/benches/benchmark.rs
+++ b/benches/benchmark.rs
@@ -23,6 +23,39 @@ fn benchmark(c: &mut Criterion) {
     c.bench_function("hcl::to_string(&Value)", |b| {
         b.iter(|| hcl::to_string(&value))
     });
+
+    let deeply_nested = r#"
+        variable "network_integration" {
+          description = "Map of networking integrations between accounts"
+          type = map(object({
+            friendly_name = string,
+            vpcs = map(object({
+              id           = string
+              cidr         = string
+              region       = string
+              description  = string
+              subnets      = map(string)
+              route_tables = map(string)
+              security_groups = map(object({
+                id = string
+                rules = map(object({
+                  direction   = string
+                  protocol    = string
+                  from_port   = string
+                  to_port     = string
+                  description = string
+                }))
+              }))
+            }))
+            additional_propagated_vpcs   = list(string)
+            additional_static_vpc_routes = list(string)
+          }))
+          default = {}
+        }"#;
+
+    c.bench_function("hcl::parse(&deeply_nested)", |b| {
+        b.iter(|| hcl::parse(&deeply_nested))
+    });
 }
 
 criterion_group!(benches, benchmark);
diff --git a/specsuite/hcl/invalid-heredoc.hcl.json b/specsuite/hcl/invalid-heredoc.hcl.json
index e968c49..ca4226f 100644
--- a/specsuite/hcl/invalid-heredoc.hcl.json
+++ b/specsuite/hcl/invalid-heredoc.hcl.json
@@ -1,6 +1,6 @@
 {
   "message": "EOF must be immediately followed by a newline",
   "body": {
-    "message": " --> 2:12\n  |\n2 |   source = <<-EOF # This is invalid\n  |            ^---\n  |\n  = expected ExprTerm, Operation, or Conditional"
+    "message": " --> 2:12\n  |\n2 |   source = <<-EOF # This is invalid\n  |            ^---\n  |\n  = expected Expression"
   }
 }
diff --git a/src/parser/grammar/hcl.pest b/src/parser/grammar/hcl.pest
index 1f10a08..5ce8db6 100644
--- a/src/parser/grammar/hcl.pest
+++ b/src/parser/grammar/hcl.pest
@@ -10,10 +10,17 @@ Block     = { Identifier ~ (StringLit | Identifier)* ~ BlockBody }
 BlockBody = { "{" ~ Body ~ "}" }
 
 // Expressions
-Expression    = _{ Conditional | Operation | ExprTerm }
-ExprTerm      =  { ExprTermInner ~ (Splat | GetAttr | Index)* }
-ExprTermInner = _{ Value | TemplateExpr | FunctionCall | Variable | ForExpr | Parenthesis }
-Parenthesis   =  { "(" ~ Expression ~ ")" }
+Expression = {
+    UnaryOperator? ~
+    ExprTerm ~
+    (BinaryOperator ~ Expression)? ~
+    ("?" ~ Expression ~ ":" ~ Expression)?
+}
+ExprTerm = {
+    (Value | TemplateExpr | FunctionCall | Variable | ForExpr | Parenthesis) ~
+    (Splat | GetAttr | Index)*
+}
+Parenthesis = { "(" ~ Expression ~ ")" }
 
 // Values
 Value = _{ LiteralValue | CollectionValue }
@@ -84,16 +91,9 @@ StringPart = {
 }
 
 // Functions and function calls
-FunctionCall = ${
-    Identifier ~ "(" ~
-    WHITESPACE* ~ (COMMENT ~ WHITESPACE*)* ~
-    Arguments ~
-    WHITESPACE* ~ (COMMENT ~ WHITESPACE*)* ~
-    ")"
-}
-
-Arguments   = !{ (Expression ~ ("," ~ Expression)* ~ ("," | ExpandFinal)?)? }
-ExpandFinal =  { "..." }
+FunctionCall = { Identifier ~ Arguments }
+Arguments    = { "(" ~ (Expression ~ ("," ~ Expression)* ~ ("," | ExpandFinal)?)? ~ ")" }
+ExpandFinal  = { "..." }
 
 // For expressions
 ForExpr       = { ForTupleExpr | ForObjectExpr }
@@ -118,21 +118,13 @@ Splat     = _{ (AttrSplat ~ GetAttr*) | (FullSplat ~ (GetAttr | Index)*) }
 AttrSplat =  { ".*" }
 FullSplat =  { "[*]" }
 
-// Operations
-Operation          = { UnaryOp | BinaryOp }
-UnaryOp            = { UnaryOperator ~ Expression }
+// Unary and binary operators
 UnaryOperator      = { "-" | "!" }
-BinaryOp           = { ExprTerm ~ BinaryOperator ~ Expression }
 BinaryOperator     = { CompareOperator | ArithmeticOperator | LogicOperator }
 CompareOperator    = { "==" | "!=" | "<=" | ">=" | "<" | ">" }
 ArithmeticOperator = { "+" | "-" | "*" | "/" | "%" }
 LogicOperator      = { "&&" | "||" }
 
-
-// Conditional operator
-CondExpr    = _{ ExprTerm | Operation }
-Conditional =  { CondExpr ~ "?" ~ Expression ~ ":" ~ Expression }
-
 // Comments
 COMMENT         = _{ InlineComment | BlockComment }
 InlineComment   = _{ ("#" | "//") ~ (!EoInlineComment ~ ANY)* }
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
index 7a2179f..adb7585 100644
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -112,60 +112,58 @@ fn parse_block_body(pair: Pair<Rule>) -> Result<Body> {
 }
 
 fn parse_expression(pair: Pair<Rule>) -> Result<Expression> {
-    match pair.as_rule() {
-        Rule::ExprTerm => parse_expr_term(pair),
-        Rule::Operation => match parse_operation(inner(pair))? {
-            Operation::Binary(binary) => Ok(Expression::from(Operation::Binary(binary))),
-            Operation::Unary(unary) => match (unary.operator, unary.expr) {
-                // Negate operations on numbers are just converted to negative numbers for
-                // convenience.
-                (UnaryOperator::Neg, Expression::Number(num)) => Ok(Expression::Number(-num)),
-                (operator, expr) => Ok(Expression::from(Operation::Unary(UnaryOp {
-                    operator,
-                    expr,
-                }))),
-            },
-        },
-        Rule::Conditional => Ok(Expression::from(parse_conditional(pair)?)),
-        rule => unexpected_rule(rule),
-    }
+    let pairs = pair.into_inner();
+    let (expr, pairs) = parse_unary_op(pairs)?;
+    let (expr, pairs) = parse_binary_op(expr, pairs)?;
+    parse_conditional(expr, pairs)
 }
 
-fn parse_conditional(pair: Pair<Rule>) -> Result<Conditional> {
-    let mut pairs = pair.into_inner();
+fn parse_unary_op(mut pairs: Pairs<Rule>) -> Result<(Expression, Pairs<Rule>)> {
+    let pair = pairs.next().unwrap();
 
-    Ok(Conditional {
-        cond_expr: parse_expression(pairs.next().unwrap())?,
-        true_expr: parse_expression(pairs.next().unwrap())?,
-        false_expr: parse_expression(pairs.next().unwrap())?,
-    })
-}
+    let expr = match pair.as_rule() {
+        Rule::UnaryOperator => {
+            let operator = UnaryOperator::from_str(pair.as_str())?;
+            let expr = parse_expr_term(pairs.next().unwrap())?;
 
-fn parse_operation(pair: Pair<Rule>) -> Result<Operation> {
-    match pair.as_rule() {
-        Rule::UnaryOp => parse_unary_op(pair).map(Operation::Unary),
-        Rule::BinaryOp => parse_binary_op(pair).map(Operation::Binary),
-        rule => unexpected_rule(rule),
-    }
+            match (operator, expr) {
+                (UnaryOperator::Neg, Expression::Number(num)) => Expression::Number(-num),
+                (operator, expr) => Expression::from(Operation::Unary(UnaryOp { operator, expr })),
+            }
+        }
+        _ => parse_expr_term(pair)?,
+    };
+
+    Ok((expr, pairs))
 }
 
-fn parse_unary_op(pair: Pair<Rule>) -> Result<UnaryOp> {
-    let mut pairs = pair.into_inner();
+fn parse_binary_op(expr: Expression, mut pairs: Pairs<Rule>) -> Result<(Expression, Pairs<Rule>)> {
+    let expr = match pairs.peek() {
+        Some(pair) => match pair.as_rule() {
+            Rule::BinaryOperator => Expression::from(Operation::Binary(BinaryOp {
+                lhs_expr: expr,
+                operator: pairs.next().unwrap().as_str().parse()?,
+                rhs_expr: parse_expression(pairs.next().unwrap())?,
+            })),
+            _ => expr,
+        },
+        None => expr,
+    };
 
-    Ok(UnaryOp {
-        operator: pairs.next().unwrap().as_str().parse()?,
-        expr: parse_expression(pairs.next().unwrap())?,
-    })
+    Ok((expr, pairs))
 }
 
-fn parse_binary_op(pair: Pair<Rule>) -> Result<BinaryOp> {
-    let mut pairs = pair.into_inner();
+fn parse_conditional(expr: Expression, mut pairs: Pairs<Rule>) -> Result<Expression> {
+    let expr = match pairs.next() {
+        Some(pair) => Expression::from(Conditional {
+            cond_expr: expr,
+            true_expr: parse_expression(pair)?,
+            false_expr: parse_expression(pairs.next().unwrap())?,
+        }),
+        None => expr,
+    };
 
-    Ok(BinaryOp {
-        lhs_expr: parse_expr_term(pairs.next().unwrap())?,
-        operator: pairs.next().unwrap().as_str().parse()?,
-        rhs_expr: parse_expression(pairs.next().unwrap())?,
-    })
+    Ok(expr)
 }
 
 fn parse_expressions(pair: Pair<Rule>) -> Result<Vec<Expression>> {
@@ -183,10 +181,7 @@ fn parse_expr_term(pair: Pair<Rule>) -> Result<Expression> {
         Rule::Int => Expression::Number(parse_primitive::<i64>(pair).into()),
         Rule::NullLit => Expression::Null,
         Rule::StringLit => parse_string(inner(pair)).map(Expression::String)?,
-        Rule::TemplateExpr => {
-            let expr = parse_template_expr(inner(pair));
-            Expression::TemplateExpr(Box::new(expr))
-        }
+        Rule::TemplateExpr => Expression::TemplateExpr(Box::new(parse_template_expr(inner(pair)))),
         Rule::Tuple => parse_expressions(pair).map(Expression::Array)?,
         Rule::Object => parse_object(pair).map(Expression::Object)?,
         Rule::Variable => Expression::Variable(parse_ident(pair).into()),
@@ -196,12 +191,18 @@ fn parse_expr_term(pair: Pair<Rule>) -> Result<Expression> {
         rule => unexpected_rule(rule),
     };
 
-    match pairs.peek() {
-        Some(_) => {
-            let operators = pairs.map(parse_traversal_operator).collect::<Result<_>>()?;
-            Ok(Expression::from(Traversal { expr, operators }))
-        }
-        None => Ok(expr),
+    parse_traversal(expr, pairs)
+}
+
+fn parse_traversal(expr: Expression, pairs: Pairs<Rule>) -> Result<Expression> {
+    let operators = pairs
+        .map(parse_traversal_operator)
+        .collect::<Result<Vec<TraversalOperator>>>()?;
+
+    if !operators.is_empty() {
+        Ok(Expression::from(Traversal { expr, operators }))
+    } else {
+        Ok(expr)
     }
 }
 


Test patch:
diff --git a/src/parser/tests.rs b/src/parser/tests.rs
index c4d32a4..1da9aaa 100644
--- a/src/parser/tests.rs
+++ b/src/parser/tests.rs
@@ -58,9 +58,11 @@ fn parse_attr() {
         tokens: [
             Attribute(0, 11, [
                 Identifier(0, 3),
-                ExprTerm(6, 11, [
-                    StringLit(6, 11, [
-                        String(7, 10)
+                Expression(6, 11, [
+                    ExprTerm(6, 11, [
+                        StringLit(6, 11, [
+                            String(7, 10)
+                        ])
                     ])
                 ])
             ])
@@ -73,20 +75,24 @@ fn conditional() {
     parses_to! {
         parser: HclParser,
         input: "var.enabled ? 1 : 0",
-        rule: Rule::Conditional,
+        rule: Rule::Expression,
         tokens: [
-            Conditional(0, 19, [
+            Expression(0, 19, [
                 ExprTerm(0, 11, [
                     Variable(0, 3),
                     GetAttr(3, 11, [
                         Identifier(4, 11)
                     ])
                 ]),
-                ExprTerm(14, 16, [
-                    Int(14, 15)
+                Expression(14, 16, [
+                    ExprTerm(14, 16, [
+                        Int(14, 15)
+                    ])
                 ]),
-                ExprTerm(18, 19, [
-                    Int(18, 19)
+                Expression(18, 19, [
+                    ExprTerm(18, 19, [
+                        Int(18, 19)
+                    ])
                 ])
             ])
         ]
@@ -127,16 +133,20 @@ resource "aws_s3_bucket" "mybucket" {
                         Body(41, 297, [
                             Attribute(41, 70, [
                                 Identifier(41, 47),
-                                ExprTerm(57, 70, [
-                                    StringLit(57, 67, [
-                                        String(58, 66)
+                                Expression(57, 70, [
+                                    ExprTerm(57, 70, [
+                                        StringLit(57, 67, [
+                                            String(58, 66)
+                                        ])
                                     ])
                                 ])
                             ]),
                             Attribute(70, 94, [
                                 Identifier(70, 83),
-                                ExprTerm(86, 94, [
-                                    BooleanLit(86, 90)
+                                Expression(86, 94, [
+                                    ExprTerm(86, 94, [
+                                        BooleanLit(86, 90)
+                                    ])
                                 ])
                             ]),
                             Block(94, 297, [
@@ -151,23 +161,27 @@ resource "aws_s3_bucket" "mybucket" {
                                                         Identifier(150, 189),
                                                         BlockBody(190, 287, [
                                                             Body(200, 286, [
-                                                                Attribute(200, 241, [
+                                                                Attribute(200, 250, [
                                                                     Identifier(200, 217),
-                                                                    ExprTerm(220, 241, [
-                                                                        Variable(220, 231),
-                                                                        GetAttr(231, 237, [
-                                                                            Identifier(232, 237)
-                                                                        ]),
-                                                                        GetAttr(237, 241, [
-                                                                            Identifier(238, 241)
+                                                                    Expression(220, 250, [
+                                                                        ExprTerm(220, 241, [
+                                                                            Variable(220, 231),
+                                                                            GetAttr(231, 237, [
+                                                                                Identifier(232, 237)
+                                                                            ]),
+                                                                            GetAttr(237, 241, [
+                                                                                Identifier(238, 241)
+                                                                            ])
                                                                         ])
                                                                     ])
                                                                 ]),
                                                                 Attribute(250, 286, [
                                                                     Identifier(250, 263),
-                                                                    ExprTerm(270, 286, [
-                                                                        StringLit(270, 279, [
-                                                                            String(271, 278)
+                                                                    Expression(270, 286, [
+                                                                        ExprTerm(270, 286, [
+                                                                            StringLit(270, 279, [
+                                                                                String(271, 278)
+                                                                            ])
                                                                         ])
                                                                     ])
                                                                 ])
@@ -197,18 +211,26 @@ fn parse_collections() {
         tokens: [
             Attribute(0, 22, [
                 Identifier(0, 3),
-                ExprTerm(6, 22, [
-                    Tuple(6, 22, [
-                        ExprTerm(7, 12, [
-                            StringLit(7, 12, [
-                                String(8, 11)
-                            ])
-                        ]),
-                        ExprTerm(14, 21, [
-                            Tuple(14, 21, [
-                                ExprTerm(15, 20, [
-                                    StringLit(15, 20, [
-                                        String(16, 19)
+                Expression(6, 22, [
+                    ExprTerm(6, 22, [
+                        Tuple(6, 22, [
+                            Expression(7, 12, [
+                                ExprTerm(7, 12, [
+                                    StringLit(7, 12, [
+                                        String(8, 11)
+                                    ])
+                                ])
+                            ]),
+                            Expression(14, 21, [
+                                ExprTerm(14, 21, [
+                                    Tuple(14, 21, [
+                                        Expression(15, 20, [
+                                            ExprTerm(15, 20, [
+                                                StringLit(15, 20, [
+                                                    String(16, 19)
+                                                ])
+                                            ])
+                                        ])
                                     ])
                                 ])
                             ])
@@ -226,25 +248,35 @@ fn parse_collections() {
         tokens: [
             Attribute(0, 36, [
                 Identifier(0, 3),
-                ExprTerm(6, 36, [
-                    Object(6, 36, [
-                        ExprTerm(7, 13, [
-                            StringLit(7, 12, [
-                                String(8, 11)
-                            ])
-                        ]),
-                        ExprTerm(15, 20, [
-                            StringLit(15, 20, [
-                                String(16, 19)
-                            ])
-                        ]),
-                        ExprTerm(21, 27, [
-                            StringLit(21, 26, [
-                                String(22, 25)
+                Expression(6, 36, [
+                    ExprTerm(6, 36, [
+                        Object(6, 36, [
+                            Expression(7, 13, [
+                                ExprTerm(7, 13, [
+                                    StringLit(7, 12, [
+                                        String(8, 11)
+                                    ])
+                                ])
+                            ]),
+                            Expression(15, 20, [
+                                ExprTerm(15, 20, [
+                                    StringLit(15, 20, [
+                                        String(16, 19)
+                                    ])
+                                ])
+                            ]),
+                            Expression(21, 27, [
+                                ExprTerm(21, 27, [
+                                    StringLit(21, 26, [
+                                        String(22, 25)
+                                    ])
+                                ])
+                            ]),
+                            Expression(29, 35, [
+                                ExprTerm(29, 35, [
+                                    Variable(29, 34)
+                                ])
                             ])
-                        ]),
-                        ExprTerm(29, 35, [
-                            Variable(29, 34)
                         ])
                     ])
                 ])
@@ -284,8 +316,10 @@ fn parse_template() {
                             QuotedStringTemplateLiteral(1, 5),
                             TemplateInterpolation(5, 11, [
                                 TemplateIExprStartNormal(5, 7),
-                                ExprTerm(7, 10, [
-                                    Variable(7, 10)
+                                Expression(7, 10, [
+                                    ExprTerm(7, 10, [
+                                        Variable(7, 10)
+                                    ])
                                 ]),
                                 TemplateExprEndNormal(10, 11)
                             ]),
@@ -294,8 +328,10 @@ fn parse_template() {
                                 TemplateIf(21, 48, [
                                     TemplateIfExpr(21, 38, [
                                         TemplateDExprStartNormal(21, 23),
-                                        ExprTerm(26, 31, [
-                                            Variable(26, 30)
+                                        Expression(26, 31, [
+                                            ExprTerm(26, 31, [
+                                                Variable(26, 30)
+                                            ])
                                         ]),
                                         TemplateExprEndStrip(31, 33),
                                         Template(34, 38, [
@@ -325,31 +361,37 @@ fn parse_cond_in_interpolation() {
         tokens: [
             Attribute(0, 37, [
                 Identifier(0, 4),
-                ExprTerm(7, 37, [
-                    TemplateExpr(7, 37, [
-                        QuotedStringTemplate(7, 37, [
-                            QuotedStringTemplateInner(8, 36, [
-                                TemplateInterpolation(8, 36, [
-                                    TemplateIExprStartNormal(8, 10),
-                                    Conditional(10, 35, [
-                                        ExprTerm(10, 15, [
-                                            Variable(10, 13),
-                                            GetAttr(13, 15, [
-                                                Identifier(14, 15)
-                                            ])
-                                        ]),
-                                        ExprTerm(18, 31, [
-                                            StringLit(18, 30, [
-                                                String(19, 29)
+                Expression(7, 37, [
+                    ExprTerm(7, 37, [
+                        TemplateExpr(7, 37, [
+                            QuotedStringTemplate(7, 37, [
+                                QuotedStringTemplateInner(8, 36, [
+                                    TemplateInterpolation(8, 36, [
+                                        TemplateIExprStartNormal(8, 10),
+                                        Expression(10, 35, [
+                                            ExprTerm(10, 15, [
+                                                Variable(10, 13),
+                                                GetAttr(13, 15, [
+                                                    Identifier(14, 15)
+                                                ])
+                                            ]),
+                                            Expression(18, 31, [
+                                                ExprTerm(18, 31, [
+                                                    StringLit(18, 30, [
+                                                        String(19, 29)
+                                                    ])
+                                                ])
+                                            ]),
+                                            Expression(33, 35, [
+                                                ExprTerm(33, 35, [
+                                                    StringLit(33, 35, [
+                                                        String(34, 34)
+                                                    ])
+                                                ])
                                             ])
                                         ]),
-                                        ExprTerm(33, 35, [
-                                            StringLit(33, 35, [
-                                                String(34, 34)
-                                            ])
-                                        ])
-                                    ]),
-                                    TemplateExprEndNormal(35, 36)
+                                        TemplateExprEndNormal(35, 36)
+                                    ])
                                 ])
                             ])
                         ])
@@ -374,30 +416,40 @@ providers = {
             Body(1, 89, [
                 Attribute(1, 89, [
                     Identifier(1, 10),
-                    ExprTerm(13, 89, [
-                        Object(13, 89, [
-                            ExprTerm(17, 33, [
-                                Variable(17, 20),
-                                GetAttr(20, 33, [
-                                    Identifier(21, 33)
+                    Expression(13, 89, [
+                        ExprTerm(13, 89, [
+                            Object(13, 89, [
+                                Expression(17, 34, [
+                                    ExprTerm(17, 33, [
+                                        Variable(17, 20),
+                                        GetAttr(20, 33, [
+                                            Identifier(21, 33)
+                                        ]),
+                                    ])
                                 ]),
-                            ]),
-                            ExprTerm(36, 52, [
-                                Variable(36, 39),
-                                GetAttr(39, 52, [
-                                    Identifier(40, 52)
+                                Expression(36, 55, [
+                                    ExprTerm(36, 52, [
+                                        Variable(36, 39),
+                                        GetAttr(39, 52, [
+                                            Identifier(40, 52)
+                                        ]),
+                                    ])
                                 ]),
-                            ]),
-                            ExprTerm(55, 68, [
-                                Variable(55, 58),
-                                GetAttr(58, 68, [
-                                    Identifier(59, 68)
+                                Expression(55, 72, [
+                                    ExprTerm(55, 68, [
+                                        Variable(55, 58),
+                                        GetAttr(58, 68, [
+                                            Identifier(59, 68)
+                                        ]),
+                                    ])
                                 ]),
-                            ]),
-                            ExprTerm(74, 87, [
-                                Variable(74, 77),
-                                GetAttr(77, 87, [
-                                    Identifier(78, 87)
+                                Expression(74, 88, [
+                                    ExprTerm(74, 87, [
+                                        Variable(74, 77),
+                                        GetAttr(77, 87, [
+                                            Identifier(78, 87)
+                                        ])
+                                    ])
                                 ])
                             ])
                         ])
@@ -417,36 +469,44 @@ fn parse_nested_function_call_with_splat() {
         tokens: [
             FunctionCall(0, 72, [
                 Identifier(0, 7),
-                Arguments(8, 71, [
-                    ExprTerm(8, 68, [
-                        FunctionCall(8, 68, [
-                            Identifier(8, 14),
-                            Arguments(15, 67, [
-                                ExprTerm(15, 40, [
-                                    Variable(15, 26),
-                                    GetAttr(26, 34, [
-                                        Identifier(27, 34)
-                                    ]),
-                                    AttrSplat(34, 36),
-                                    GetAttr(36, 40, [
-                                        Identifier(37, 40)
-                                    ]),
-                                ]),
-                                ExprTerm(42, 67, [
-                                    Variable(42, 53),
-                                    GetAttr(53, 61, [
-                                        Identifier(54, 61)
+                Arguments(7, 72, [
+                    Expression(8, 68, [
+                        ExprTerm(8, 68, [
+                            FunctionCall(8, 68, [
+                                Identifier(8, 14),
+                                Arguments(14, 68, [
+                                    Expression(15, 40, [
+                                        ExprTerm(15, 40, [
+                                            Variable(15, 26),
+                                            GetAttr(26, 34, [
+                                                Identifier(27, 34)
+                                            ]),
+                                            AttrSplat(34, 36),
+                                            GetAttr(36, 40, [
+                                                Identifier(37, 40)
+                                            ]),
+                                        ])
                                     ]),
-                                    AttrSplat(61, 63),
-                                    GetAttr(63, 67, [
-                                        Identifier(64, 67)
+                                    Expression(42, 67, [
+                                        ExprTerm(42, 67, [
+                                            Variable(42, 53),
+                                            GetAttr(53, 61, [
+                                                Identifier(54, 61)
+                                            ]),
+                                            AttrSplat(61, 63),
+                                            GetAttr(63, 67, [
+                                                Identifier(64, 67)
+                                            ])
+                                        ])
                                     ])
                                 ])
                             ])
                         ])
                     ]),
-                    ExprTerm(70, 71, [
-                        Int(70, 71)
+                    Expression(70, 71, [
+                        ExprTerm(70, 71, [
+                            Int(70, 71)
+                        ])
                     ])
                 ])
             ])
@@ -463,14 +523,14 @@ fn parse_traversal_with_expression() {
         tokens: [
             Attribute(0, 86, [
                 Identifier(0, 14),
-                ExprTerm(17, 86, [
-                    Variable(17, 32),
-                    GetAttr(32, 40, [
-                        Identifier(33, 40)
-                    ]),
-                    Index(40, 83, [
-                        Operation(41, 82, [
-                            BinaryOp(41, 82, [
+                Expression(17, 86, [
+                    ExprTerm(17, 86, [
+                        Variable(17, 32),
+                        GetAttr(32, 40, [
+                            Identifier(33, 40)
+                        ]),
+                        Index(40, 83, [
+                            Expression(41, 82, [
                                 ExprTerm(41, 52, [
                                     Variable(41, 46),
                                     GetAttr(46, 52, [
@@ -480,17 +540,19 @@ fn parse_traversal_with_expression() {
                                 BinaryOperator(53, 54, [
                                     ArithmeticOperator(53, 54)
                                 ]),
-                                ExprTerm(55, 82, [
-                                    Variable(55, 58),
-                                    GetAttr(58, 82, [
-                                        Identifier(59, 82)
-                                    ]),
+                                Expression(55, 82, [
+                                    ExprTerm(55, 82, [
+                                        Variable(55, 58),
+                                        GetAttr(58, 82, [
+                                            Identifier(59, 82)
+                                        ]),
+                                    ])
                                 ])
                             ])
+                        ]),
+                        GetAttr(83, 86, [
+                            Identifier(84, 86)
                         ])
-                    ]),
-                    GetAttr(83, 86, [
-                        Identifier(84, 86)
                     ])
                 ])
             ])
@@ -507,8 +569,10 @@ fn parse_null_in_variable_expr() {
         tokens: [
             Attribute(0, 14, [
                 Identifier(0, 3),
-                ExprTerm(6, 14, [
-                    Variable(6, 14)
+                Expression(6, 14, [
+                    ExprTerm(6, 14, [
+                        Variable(6, 14)
+                    ])
                 ])
             ])
         ]