Repository: lark-parser/lark
Pull request #770 (2020-11-20 23:20:26)

Added `lark.lark` to the standardlib of grammars.
This moves `lark.lark` into the `lark.grammars` folders. This means that it will be shiped with lark and can be used by packages that want to do something with lark grammars, like `lark-railroad` or the language server.

Since it is now in an even more official place than before, I added a check that tries to parse every grammar used inside the `_make_parser_tests`. This forces us to keep the grammar up-to-date. (as example, I already had to fix something in the grammar since we introduces multi line regex.) The cost of this check is minimal, but not zero. It adds (on my machine) ~2.5% to the total runtime.


Patch:
diff --git a/examples/lark_grammar.py b/examples/lark_grammar.py
index f2185ee..6730db9 100644
--- a/examples/lark_grammar.py
+++ b/examples/lark_grammar.py
@@ -7,13 +7,13 @@ A reference implementation of the Lark grammar (using LALR(1))
 import lark
 from pathlib import Path
 
-parser = lark.Lark.open('lark.lark', rel_to=__file__, parser="lalr")
-
 examples_path = Path(__file__).parent
 lark_path = Path(lark.__file__).parent
 
+parser = lark.Lark.open(lark_path / 'grammars/lark.lark', rel_to=__file__, parser="lalr")
+
+
 grammar_files = [
-    examples_path / 'lark.lark',
     examples_path / 'advanced/python2.lark',
     examples_path / 'advanced/python3.lark',
     examples_path / 'relative-imports/multiples.lark',
@@ -21,7 +21,11 @@ grammar_files = [
     examples_path / 'relative-imports/multiple3.lark',
     examples_path / 'tests/no_newline_at_end.lark',
     examples_path / 'tests/negative_priority.lark',
+    examples_path / 'standalone/json.lark',
     lark_path / 'grammars/common.lark',
+    lark_path / 'grammars/lark.lark',
+    lark_path / 'grammars/unicode.lark',
+    lark_path / 'grammars/python.lark',
 ]
 
 def test():
diff --git a/lark-stubs/lark.pyi b/lark-stubs/lark.pyi
index 7dc8626..8363a5d 100644
--- a/lark-stubs/lark.pyi
+++ b/lark-stubs/lark.pyi
@@ -63,7 +63,7 @@ class Lark:
         *,
         start: Union[None, str, List[str]] = "start",
         parser: Literal["earley", "lalr", "cyk"] = "auto",
-        lexer: Union[Literal["auto", "standard", "contextual", "dynamic", "dynamic_complete"], Type[Lexer]] = "auto",
+        lexer: Union[Literal["auto", "standard", "contextual", "dynamic", "dynamic_complete"], Lexer] = "auto",
         transformer: Optional[Transformer] = None,
         postlex: Optional[PostLex] = None,
         ambiguity: Literal["explicit", "resolve"] = "resolve",
diff --git a/lark-stubs/lexer.pyi b/lark-stubs/lexer.pyi
index a654b0f..12d3dfe 100644
--- a/lark-stubs/lexer.pyi
+++ b/lark-stubs/lexer.pyi
@@ -85,9 +85,6 @@ class Token(str):
     end_column: int
     end_pos: int
 
-    def __init__(self, type_: str, value: Any, pos_in_stream: int = None, line: int = None, column: int = None, end_line: int = None, end_column: int = None, end_pos: int = None):
-        ...
-
     def update(self, type_: Optional[str] = None, value: Optional[str] = None) -> Token:
         ...
 
diff --git a/examples/lark.lark b/lark/grammars/lark.lark
similarity index 96%
rename from examples/lark.lark
rename to lark/grammars/lark.lark
index 474b819..c16db47 100644
--- a/examples/lark.lark
+++ b/lark/grammars/lark.lark
@@ -45,7 +45,7 @@ OP: /[+*]|[?](?![a-z])/
 RULE: /!?[_?]?[a-z][_a-z0-9]*/
 TOKEN: /_?[A-Z][_A-Z0-9]*/
 STRING: _STRING "i"?
-REGEXP: /\/(?!\/)(\\\/|\\\\|[^\/\n])*?\/[imslux]*/
+REGEXP: /\/(?!\/)(\\\/|\\\\|[^\/])*?\/[imslux]*/
 _NL: /(\r?\n)+\s*/
 
 %import common.ESCAPED_STRING -> _STRING


Test patch:
diff --git a/tests/__main__.py b/tests/__main__.py
index 1814564..5ec89e3 100644
--- a/tests/__main__.py
+++ b/tests/__main__.py
@@ -9,7 +9,6 @@ from .test_tools import TestStandalone
 from .test_cache import TestCache
 from .test_grammar import TestGrammar
 from .test_reconstructor import TestReconstructor
-from .test_tree_forest_transformer import TestTreeForestTransformer
 
 try:
     from .test_nearley.test_nearley import TestNearley
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 39bd00c..1d9827c 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -30,6 +30,7 @@ try:
 except ImportError:
     regex = None
 
+import lark
 from lark import logger
 from lark.lark import Lark
 from lark.exceptions import GrammarError, ParseError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters
@@ -942,10 +943,24 @@ class DualBytesLark:
 
 def _make_parser_test(LEXER, PARSER):
     lexer_class_or_name = CustomLexer if LEXER == 'custom' else LEXER
-    def _Lark(grammar, **kwargs):
-        return Lark(grammar, lexer=lexer_class_or_name, parser=PARSER, propagate_positions=True, **kwargs)
-    def _Lark_open(gfilename, **kwargs):
-        return Lark.open(gfilename, lexer=lexer_class_or_name, parser=PARSER, propagate_positions=True, **kwargs)
+    if (LEXER, PARSER) == ('standard', 'earley'): 
+        # Check that the `lark.lark` grammar represents can parse every example used in these tests.
+        # Doesn't matter when we do it, or how often. Just do it for the first entry in `TO_TEST`
+        lalr_parser = Lark.open(os.path.join(os.path.dirname(lark.__file__), 'grammars/lark.lark'), parser='lalr')
+        def _Lark(grammar, **kwargs):
+            inst = Lark(grammar, lexer=lexer_class_or_name, parser=PARSER, propagate_positions=True, **kwargs)
+            lalr_parser.parse(inst.source_grammar) # Test after instance creation. When the grammar should fail, don't test it.
+            return inst
+        def _Lark_open(gfilename, **kwargs):
+            inst = Lark.open(gfilename, lexer=lexer_class_or_name, parser=PARSER, propagate_positions=True, **kwargs)
+            lalr_parser.parse(inst.source_grammar)
+            return inst
+    else:
+        def _Lark(grammar, **kwargs):
+            return Lark(grammar, lexer=lexer_class_or_name, parser=PARSER, propagate_positions=True, **kwargs)
+        def _Lark_open(gfilename, **kwargs):
+            return Lark.open(gfilename, lexer=lexer_class_or_name, parser=PARSER, propagate_positions=True, **kwargs)
+
 
     class _TestParser(unittest.TestCase):
         def test_basic1(self):