Repository: hylo-lang/hylo (swift)

Issue:
Support trailing separators in compound literals
The parse rejects `[a, b,]` and `[a:1, b:2,]`.

Pull request:
Resolves #129. Following @kyouko-taiga's gradual migration away from Durian, I've rewritten the parsers for map and array literals in a more imperative form, and added tests to verify that trailing commas parse successfully.

Patch:
diff --git a/Sources/FrontEnd/Parse/Parser.swift b/Sources/FrontEnd/Parse/Parser.swift
index 94cace7a7..5f8199c7c 100644
--- a/Sources/FrontEnd/Parse/Parser.swift
+++ b/Sources/FrontEnd/Parse/Parser.swift
@@ -2313,53 +2313,21 @@ public enum Parser {
   }
 
   private static func parseCompoundLiteral(in state: inout ParserState) throws -> AnyExprID? {
-    let backup = state.backup()
-
-    // Attempt to parse a map literal.
-    do {
-      if let expr = try mapLiteral.parse(&state) { return AnyExprID(expr) }
-    } catch {}
-
-    // Backtrack and parse a buffer literal.
-    state.restore(from: backup)
-    return try bufferLiteral.parse(&state).map(AnyExprID.init)
-  }
-
-  private static let bufferLiteral =
-    (take(.lBrack).and(maybe(bufferComponentListContents)).and(take(.rBrack))
-      .map({ (state, tree) -> BufferLiteralExpr.ID in
-        state.insert(
-          BufferLiteralExpr(
-            elements: tree.0.1 ?? [],
-            site: tree.0.0.site.extended(upTo: state.currentIndex)))
-      }))
-
-  private static let bufferComponentListContents =
-    (expr.and(zeroOrMany(take(.comma).and(expr).second))
-      .map({ (state, tree) -> [AnyExprID] in [tree.0] + tree.1 }))
-
-  private static let mapLiteral =
-    (take(.lBrack).and(mapComponentListContents.or(mapComponentEmptyContents)).and(take(.rBrack))
-      .map({ (state, tree) -> MapLiteralExpr.ID in
-        state.insert(
-          MapLiteralExpr(
-            elements: tree.0.1,
-            site: tree.0.0.site.extended(upTo: state.currentIndex)))
-      }))
+    if let map = try parseMapLiteral(in: &state) {
+      return AnyExprID(state.insert(map))
+    }
 
-  private static let mapComponentEmptyContents =
-    (take(.colon)
-      .map({ (_, _) -> [MapLiteralExpr.Element] in [] }))
+    if let buffer = try bufferLiteral.parse(&state) {
+      let expr = state.insert(
+        BufferLiteralExpr(
+          elements: buffer.elements,
+          site: state.range(from: buffer.opener.site.start)))
+      return AnyExprID(expr)
+    }
 
-  private static let mapComponentListContents =
-    (mapComponent.and(zeroOrMany(take(.comma).and(mapComponent).second))
-      .map({ (state, tree) -> [MapLiteralExpr.Element] in [tree.0] + tree.1 }))
+    return nil
 
-  private static let mapComponent =
-    (expr.and(take(.colon)).and(expr)
-      .map({ (_, tree) -> MapLiteralExpr.Element in
-        MapLiteralExpr.Element(key: tree.0.0, value: tree.1)
-      }))
+  }
 
   private static let callArgument = Apply(parseArgument(in:))
 
@@ -2481,6 +2449,50 @@ public enum Parser {
     closerDescription: "}",
     elementParser: Apply(parseTupleTypeExprElement(in:)))
 
+  private static let bufferLiteral = DelimitedCommaSeparatedList(
+    openerKind: .lBrack,
+    closerKind: .rBrack,
+    closerDescription: "]",
+    elementParser: Apply(parseExpr(in:)))
+
+  private static let nonemptyMapLiteral = DelimitedCommaSeparatedList(
+    openerKind: .lBrack,
+    closerKind: .rBrack,
+    closerDescription: "]",
+    elementParser: Apply(parseMapElement(in:)))
+
+  private static func parseMapElement(in state: inout ParserState) throws -> MapLiteralExpr.Element?
+  {
+    let backup = state.backup()
+
+    if let lhs = try parseExpr(in: &state) {
+      if state.take(.colon) != nil {
+        if let rhs = try parseExpr(in: &state) {
+          return MapLiteralExpr.Element(key: AnyExprID(lhs), value: AnyExprID(rhs))
+        }
+      }
+    }
+
+    state.restore(from: backup)
+    return nil
+  }
+
+  private static func parseMapLiteral(in state: inout ParserState) throws -> MapLiteralExpr? {
+    if let mapLiteral = state.take(.lBrack, .colon, .rBrack) {
+      return MapLiteralExpr(elements: [], site: mapLiteral.first!.site)
+    }
+
+    let backup = state.backup()
+
+    if let expr = try nonemptyMapLiteral.parse(&state), expr.closer != nil {
+      return MapLiteralExpr(elements: expr.elements, site: expr.opener.site)
+    }
+
+    state.restore(from: backup)
+
+    return nil
+  }
+
   // MARK: Patterns
 
   private static func anyPattern<Base: Combinator>(
diff --git a/Sources/FrontEnd/Parse/ParserState.swift b/Sources/FrontEnd/Parse/ParserState.swift
index 85270703c..245ed5e86 100644
--- a/Sources/FrontEnd/Parse/ParserState.swift
+++ b/Sources/FrontEnd/Parse/ParserState.swift
@@ -139,6 +139,15 @@ struct ParserState {
     return token
   }
 
+  /// Fills the lookahead buffer until it contains `n` tokens, or fewer if the lexer is exhausted.
+  mutating func peek(_ n: Int) -> Deque<Token>.SubSequence {
+    while lookahead.count < n {
+      guard let t = lexer.next() else { break }
+      lookahead.append(t)
+    }
+    return lookahead.prefix(upTo: min(n, lookahead.count))
+  }
+
   /// Returns whether a token of the given `kind` is next in the input.
   mutating func isNext(_ kind: Token.Kind) -> Bool {
     peek()?.kind == kind
@@ -178,6 +187,19 @@ struct ParserState {
     }
   }
 
+  /// Consumes and returns the first `kinds.count` tokens if they have the specified kinds.
+  mutating func take(_ kinds: Token.Kind...) -> [Token]? {
+    let tokens = peek(kinds.count)
+
+    if tokens.elementsEqual(kinds, by: { (a, b) in a.kind == b }) {
+      lookahead.removeFirst(kinds.count)
+      currentIndex = tokens.last!.site.end
+      return Array(tokens)
+    }
+
+    return nil
+  }
+
   /// Consumes and returns the next token, only if it has the specified kind and if it is not
   /// preceeded by any whitespace.
   mutating func takeWithoutSkippingWhitespace(_ kind: Token.Kind) -> Token? {


Test patch:
diff --git a/Tests/ValTests/ParserTests.swift b/Tests/ValTests/ParserTests.swift
index 39e451b43..e1d920319 100644
--- a/Tests/ValTests/ParserTests.swift
+++ b/Tests/ValTests/ParserTests.swift
@@ -1105,6 +1105,13 @@ final class ParserTests: XCTestCase {
     XCTAssertEqual(expr.elements.count, 3)
   }
 
+  func testBufferLiteralWithTrailingComma() throws {
+    let input: SourceFile = "[a, b, c,]"
+    let (exprID, ast) = try input.parse(with: Parser.parseExpr(in:))
+    let expr = try XCTUnwrap(ast[exprID] as? BufferLiteralExpr)
+    XCTAssertEqual(expr.elements.count, 3)
+  }
+
   func testMapLiteral() throws {
     let input: SourceFile = "[:]"
     let (exprID, ast) = try input.parse(with: Parser.parseExpr(in:))
@@ -1119,6 +1126,13 @@ final class ParserTests: XCTestCase {
     XCTAssertEqual(expr.elements.count, 3)
   }
 
+  func testMapLiteralWithTrailingComma() throws {
+    let input: SourceFile = "[a: 0, b: 1, c: 2,]"
+    let (exprID, ast) = try input.parse(with: Parser.parseExpr(in:))
+    let expr = try XCTUnwrap(ast[exprID] as? MapLiteralExpr)
+    XCTAssertEqual(expr.elements.count, 3)
+  }
+
   func testPrimaryDeclRefSansHint() throws {
     let input: SourceFile = "foo<T, size: 42>"
     let (exprID, ast) = try input.parse(with: Parser.parseExpr(in:))