Repository: lark-parser/lark
Pull request #1079 (2022-01-15 13:42:18)

Bugfix: maybe_placeholders now don't accumulate over the | operator
(Fix for Issue #1078  )


Patch:
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index d06e88c..7652b44 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -193,6 +193,35 @@ SMALL_FACTOR_THRESHOLD = 5
 REPEAT_BREAK_THRESHOLD = 50
 
 
+class FindRuleSize(Transformer):
+    def __init__(self, keep_all_tokens):
+        self.keep_all_tokens = keep_all_tokens
+
+    def _will_not_get_removed(self, sym):
+        if isinstance(sym, NonTerminal):
+            return not sym.name.startswith('_')
+        if isinstance(sym, Terminal):
+            return self.keep_all_tokens or not sym.filter_out
+        if sym is _EMPTY:
+            return False
+        assert False, sym
+
+    def _args_as_int(self, args):
+        for a in args:
+            if isinstance(a, int):
+                yield a
+            elif isinstance(a, Symbol):
+                yield 1 if self._will_not_get_removed(a) else 0
+            else:
+                assert False
+
+    def expansion(self, args):
+        return sum(self._args_as_int(args))
+
+    def expansions(self, args):
+        return max(self._args_as_int(args))
+
+
 @inline_args
 class EBNF_to_BNF(Transformer_InPlace):
     def __init__(self):
@@ -344,18 +373,8 @@ class EBNF_to_BNF(Transformer_InPlace):
 
     def maybe(self, rule):
         keep_all_tokens = self.rule_options and self.rule_options.keep_all_tokens
-
-        def will_not_get_removed(sym):
-            if isinstance(sym, NonTerminal):
-                return not sym.name.startswith('_')
-            if isinstance(sym, Terminal):
-                return keep_all_tokens or not sym.filter_out
-            if sym is _EMPTY:
-                return False
-            assert False, sym
-
-        empty = ST('expansion', [_EMPTY] * len(list(rule.scan_values(will_not_get_removed))))
-
+        rule_size = FindRuleSize(keep_all_tokens).transform(rule)
+        empty = ST('expansion', [_EMPTY] * rule_size)
         return ST('expansions', [rule, empty])
 
 


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 4a8c05d..dce215a 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -3,7 +3,6 @@ from __future__ import absolute_import
 
 import re
 import unittest
-import logging
 import os
 import sys
 from copy import copy, deepcopy
@@ -2355,6 +2354,15 @@ def _make_parser_test(LEXER, PARSER):
             self.assertEqual(p.parse("a").children, ['a', None, None])
             self.assertEqual(p.parse("abc").children, ['a', 'b', 'c'])
 
+            p = _Lark(r"""!start: "a" ["b" | "c"] """, maybe_placeholders=True)
+            self.assertEqual(p.parse("a").children, ['a', None])
+            self.assertEqual(p.parse("ab").children, ['a', 'b'])
+
+            p = _Lark(r"""!start: "a" ["b" | "c" "d"] """, maybe_placeholders=True)
+            self.assertEqual(p.parse("a").children, ['a', None, None])
+            # self.assertEqual(p.parse("ab").children, ['a', 'b', None])        # Not implemented; current behavior is incorrect
+            self.assertEqual(p.parse("acd").children, ['a', 'c', 'd'])
+
 
         def test_escaped_string(self):
             "Tests common.ESCAPED_STRING"