Repository: lark-parser/lark
Pull request #748 (2020-11-10 15:41:14)

Reconstruct with unicode terminals
This test checks that a parse tree built with a grammar containing only ascii characters can be reconstructed with a grammar that has unicode rules (or vice versa). The original bug assigned ANON terminals to unicode keywords, which offsets the ANON terminal count in the unicode grammar and causes subsequent identical ANON tokens (e.g., +=) to mis-match between the two grammars.

I know that this is an uncommon situation, but it is central to use of Lark as a transpiler (e.g., [லஸ்ஸி](https://லஸ்ஸி.இந்தியா/)) and the fix was quite simple, so I thought I'd try to send a pull request. Hope you like it!


Patch:
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index eb0273c..ee38dc8 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -6,7 +6,7 @@ from copy import copy, deepcopy
 from io import open
 import pkgutil
 
-from .utils import bfs, eval_escaping, Py36, logger, classify_bool
+from .utils import bfs, eval_escaping, Py36, logger, classify_bool, is_id_continue, is_id_start
 from .lexer import Token, TerminalDef, PatternStr, PatternRE
 
 from .parse_tree_builder import ParseTreeBuilder
@@ -328,10 +328,8 @@ class PrepareAnonTerminals(Transformer_InPlace):
                 try:
                     term_name = _TERMINAL_NAMES[value]
                 except KeyError:
-                    if value.isalnum() and value[0].isalpha() and value.upper() not in self.term_set:
-                        with suppress(UnicodeEncodeError):
-                            value.upper().encode('ascii') # Make sure we don't have unicode in our terminal names
-                            term_name = value.upper()
+                    if is_id_continue(value) and is_id_start(value[0]) and value.upper() not in self.term_set:
+                        term_name = value.upper()
 
                 if term_name in self.term_set:
                     term_name = None
diff --git a/lark/reconstruct.py b/lark/reconstruct.py
index e7cff31..2efc0ae 100644
--- a/lark/reconstruct.py
+++ b/lark/reconstruct.py
@@ -8,6 +8,7 @@ from .lexer import Token, PatternStr
 from .grammar import Terminal, NonTerminal
 
 from .tree_matcher import TreeMatcher, is_discarded_terminal
+from .utils import is_id_continue
 
 def is_iter_empty(i):
     try:
@@ -56,10 +57,6 @@ class WriteTokensTransformer(Transformer_InPlace):
         return to_write
 
 
-def _isalnum(x):
-    # Categories defined here: https://www.python.org/dev/peps/pep-3131/
-    return unicodedata.category(x) in ['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl', 'Mn', 'Mc', 'Nd', 'Pc']
-
 class Reconstructor(TreeMatcher):
     """
     A Reconstructor that will, given a full parse Tree, generate source code.
@@ -97,7 +94,7 @@ class Reconstructor(TreeMatcher):
         y = []
         prev_item = ''
         for item in x:
-            if prev_item and item and _isalnum(prev_item[-1]) and _isalnum(item[0]):
+            if prev_item and item and is_id_continue(prev_item[-1]) and is_id_continue(item[0]):
                 y.append(' ')
             y.append(item)
             prev_item = item
diff --git a/lark/utils.py b/lark/utils.py
index cfd4306..89498c6 100644
--- a/lark/utils.py
+++ b/lark/utils.py
@@ -1,4 +1,5 @@
 import sys
+import unicodedata
 import os
 from functools import reduce
 from ast import literal_eval
@@ -12,6 +13,25 @@ logger.addHandler(logging.StreamHandler())
 # By default, we should not output any log messages
 logger.setLevel(logging.CRITICAL)
 
+def is_id_continue(x):
+    """
+    Checks if all characters in `x` are alphanumeric characters (Unicode standard, so diactrics, Indian vowels, non-latin
+    numbers, etc. all pass). Synonymous with a Python `ID_CONTINUE` identifier. See PEP 3131 for details.
+    """
+    if len(x) != 1:
+        return all(is_id_continue(y) for y in x)
+    return x == '_' or unicodedata.category(x) in ['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl', 'Mn', 'Mc', 'Nd', 'Pc']
+
+
+def is_id_start(x):
+    """
+    Checks if all characters in `x` are alphabetic characters (Unicode standard, so diactrics, Indian vowels, non-latin
+    numbers, etc. all pass). Synonymous with a Python `ID_START` identifier. See PEP 3131 for details.
+    """
+    if len(x) != 1:
+        return all(is_id_start(y) for y in x)
+    return x == '_' or unicodedata.category(x) in ['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Mn', 'Mc', 'Pc']
+
 
 def classify(seq, key=None, value=None):
     d = {}


Test patch:
diff --git a/tests/test_nearley/nearley b/tests/test_nearley/nearley
index a46b374..cf8925f 160000
--- a/tests/test_nearley/nearley
+++ b/tests/test_nearley/nearley
@@ -1,1 +1,1 @@
-Subproject commit a46b37471db486db0f6e1ce6a2934fb238346b44
+Subproject commit cf8925f729bde741a3076c5856c0c0862bc7f5de
diff --git a/tests/test_reconstructor.py b/tests/test_reconstructor.py
index 93c64fe..f132312 100644
--- a/tests/test_reconstructor.py
+++ b/tests/test_reconstructor.py
@@ -1,17 +1,22 @@
+# coding=utf-8
+
 import json
+import sys
 import unittest
 from unittest import TestCase
+
 from lark import Lark
 from lark.reconstruct import Reconstructor
 
-
 common = """
 %import common (WS_INLINE, NUMBER, WORD)
 %ignore WS_INLINE
 """
 
+
 def _remove_ws(s):
-    return s.replace(' ', '').replace('\n','')
+    return s.replace(' ', '').replace('\n', '')
+
 
 class TestReconstructor(TestCase):
 
@@ -22,7 +27,6 @@ class TestReconstructor(TestCase):
         self.assertEqual(_remove_ws(code), _remove_ws(new))
 
     def test_starred_rule(self):
-
         g = """
         start: item*
         item: NL
@@ -38,7 +42,6 @@ class TestReconstructor(TestCase):
         self.assert_reconstruct(g, code)
 
     def test_starred_group(self):
-
         g = """
         start: (rule | NL)*
         rule: WORD ":" NUMBER
@@ -52,7 +55,6 @@ class TestReconstructor(TestCase):
         self.assert_reconstruct(g, code)
 
     def test_alias(self):
-
         g = """
         start: line*
         line: NL
@@ -140,6 +142,43 @@ class TestReconstructor(TestCase):
         new_json = Reconstructor(json_parser).reconstruct(tree)
         self.assertEqual(json.loads(new_json), json.loads(test_json))
 
+    @unittest.skipIf(sys.version_info < (3, 0), "Python 2 does not play well with Unicode.")
+    def test_switch_grammar_unicode_terminal(self):
+        """
+        This test checks that a parse tree built with a grammar containing only ascii characters can be reconstructed
+        with a grammar that has unicode rules (or vice versa). The original bug assigned ANON terminals to unicode
+        keywords, which offsets the ANON terminal count in the unicode grammar and causes subsequent identical ANON
+        tokens (e.g., `+=`) to mis-match between the two grammars.
+        """
+
+        g1 = """
+        start: (NL | stmt)*
+        stmt: "keyword" var op var
+        !op: ("+=" | "-=" | "*=" | "/=")
+        var: WORD
+        NL: /(\\r?\\n)+\s*/
+        """ + common
+
+        g2 = """
+        start: (NL | stmt)*
+        stmt: "குறிப்பு" var op var
+        !op: ("+=" | "-=" | "*=" | "/=")
+        var: WORD
+        NL: /(\\r?\\n)+\s*/
+        """ + common
+
+        code = """
+        keyword x += y
+        """
+
+        l1 = Lark(g1, parser='lalr')
+        l2 = Lark(g2, parser='lalr')
+        r = Reconstructor(l2)
+
+        tree = l1.parse(code)
+        code2 = r.reconstruct(tree)
+        assert l2.parse(code2) == tree
+
 
 if __name__ == '__main__':
     unittest.main()