Repository: lark-parser/lark
Pull request #1123 (2022-03-06 17:44:39)

Add match stmt support to python.lark
Based on the PR of the same name by MegaIng ([PR #1016](https://github.com/lark-parser/lark/pull/1016)), but now with tests!


Patch:
diff --git a/examples/advanced/python_parser.py b/examples/advanced/python_parser.py
index 78f8520..776f6e1 100644
--- a/examples/advanced/python_parser.py
+++ b/examples/advanced/python_parser.py
@@ -40,9 +40,9 @@ def _read(fn, *args):
 def _get_lib_path():
     if os.name == 'nt':
         if 'PyPy' in sys.version:
-            return os.path.join(sys.prefix, 'lib-python', sys.winver)
+            return os.path.join(sys.base_prefix, 'lib-python', sys.winver)
         else:
-            return os.path.join(sys.prefix, 'Lib')
+            return os.path.join(sys.base_prefix, 'Lib')
     else:
         return [x for x in sys.path if x.endswith('%s.%s' % sys.version_info[:2])][0]
 
@@ -76,5 +76,4 @@ def test_earley_equals_lalr():
 if __name__ == '__main__':
     test_python_lib()
     # test_earley_equals_lalr()
-    # python_parser3.parse(_read(sys.argv[1]) + '\n')
-
+    # python_parser3.parse(_read(sys.argv[1]) + '\n')
\ No newline at end of file
diff --git a/lark/grammars/python.lark b/lark/grammars/python.lark
index 4ac80b0..8286c85 100644
--- a/lark/grammars/python.lark
+++ b/lark/grammars/python.lark
@@ -20,7 +20,7 @@ decorators: decorator+
 decorated: decorators (classdef | funcdef | async_funcdef)
 
 async_funcdef: "async" funcdef
-funcdef: "def" NAME "(" [parameters] ")" ["->" test] ":" suite
+funcdef: "def" name "(" [parameters] ")" ["->" test] ":" suite
 
 parameters: paramvalue ("," paramvalue)* ["," SLASH ("," paramvalue)*] ["," [starparams | kwparams]]
           | starparams
@@ -34,7 +34,7 @@ poststarparams: ("," paramvalue)* ["," kwparams]
 kwparams: "**" typedparam ","?
 
 ?paramvalue: typedparam ("=" test)?
-?typedparam: NAME (":" test)?
+?typedparam: name (":" test)?
 
 
 lambdef: "lambda" [lambda_params] ":" test
@@ -42,9 +42,9 @@ lambdef_nocond: "lambda" [lambda_params] ":" test_nocond
 lambda_params: lambda_paramvalue ("," lambda_paramvalue)* ["," [lambda_starparams | lambda_kwparams]]
           | lambda_starparams
           | lambda_kwparams
-?lambda_paramvalue: NAME ("=" test)?
-lambda_starparams: "*" [NAME]  ("," lambda_paramvalue)* ["," [lambda_kwparams]]
-lambda_kwparams: "**" NAME ","?
+?lambda_paramvalue: name ("=" test)?
+lambda_starparams: "*" [name]  ("," lambda_paramvalue)* ["," [lambda_kwparams]]
+lambda_kwparams: "**" name ","?
 
 
 ?stmt: simple_stmt | compound_stmt
@@ -75,16 +75,17 @@ import_name: "import" dotted_as_names
 // note below: the ("." | "...") is necessary because "..." is tokenized as ELLIPSIS
 import_from: "from" (dots? dotted_name | dots) "import" ("*" | "(" import_as_names ")" | import_as_names)
 !dots: "."+
-import_as_name: NAME ["as" NAME]
-dotted_as_name: dotted_name ["as" NAME]
+import_as_name: name ["as" name]
+dotted_as_name: dotted_name ["as" name]
 import_as_names: import_as_name ("," import_as_name)* [","]
 dotted_as_names: dotted_as_name ("," dotted_as_name)*
-dotted_name: NAME ("." NAME)*
-global_stmt: "global" NAME ("," NAME)*
-nonlocal_stmt: "nonlocal" NAME ("," NAME)*
+dotted_name: name ("." name)*
+global_stmt: "global" name ("," name)*
+nonlocal_stmt: "nonlocal" name ("," name)*
 assert_stmt: "assert" test ["," test]
 
-?compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+?compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | match_stmt
+              | with_stmt | funcdef | classdef | decorated | async_stmt
 async_stmt: "async" (funcdef | with_stmt | for_stmt)
 if_stmt: "if" test ":" suite elifs ["else" ":" suite]
 elifs: elif_*
@@ -95,19 +96,68 @@ try_stmt: "try" ":" suite except_clauses ["else" ":" suite] [finally]
         | "try" ":" suite finally   -> try_finally
 finally: "finally" ":" suite
 except_clauses: except_clause+ 
-except_clause: "except" [test ["as" NAME]] ":" suite
+except_clause: "except" [test ["as" name]] ":" suite
+// NB compile.c makes sure that the default except clause is last
+
 
 with_stmt: "with" with_items ":" suite
 with_items: with_item ("," with_item)* 
-with_item: test ["as" expr]
-// NB compile.c makes sure that the default except clause is last
+with_item: test ["as" name]
+
+match_stmt: "match" test ":" _NEWLINE _INDENT case+ _DEDENT
+
+case: "case" pattern ["if" test] ":" suite
+
+?pattern: sequence_item_pattern "," _sequence_pattern -> sequence_pattern
+        | as_pattern
+?as_pattern: or_pattern ("as" NAME)?
+?or_pattern: closed_pattern ("|" closed_pattern)*
+?closed_pattern: literal_pattern
+               | NAME -> capture_pattern
+               | "_" -> any_pattern
+               | attr_pattern
+               | "(" as_pattern ")"
+               | "[" _sequence_pattern "]" -> sequence_pattern
+               | "(" (sequence_item_pattern "," _sequence_pattern)? ")" -> sequence_pattern
+               | "{" (mapping_item_pattern ("," mapping_item_pattern)* ","?)?"}" -> mapping_pattern
+               | "{" (mapping_item_pattern ("," mapping_item_pattern)* ",")? "**" NAME ","? "}" -> mapping_star_pattern
+               | class_pattern
+
+literal_pattern: inner_literal_pattern
+
+?inner_literal_pattern: "None" -> const_none
+                      | "True" -> const_true
+                      | "False" -> const_false
+                      | STRING -> string
+                      | number
+
+attr_pattern: NAME ("." NAME)+ -> value
+
+name_or_attr_pattern: NAME ("." NAME)* -> value
+
+mapping_item_pattern: (literal_pattern|attr_pattern) ":" as_pattern
+
+_sequence_pattern: (sequence_item_pattern ("," sequence_item_pattern)* ","?)?
+?sequence_item_pattern: as_pattern
+                      | "*" NAME -> star_pattern
+
+class_pattern: name_or_attr_pattern "(" [arguments_pattern ","?] ")"
+arguments_pattern: pos_arg_pattern ["," keyws_arg_pattern]
+                 | keyws_arg_pattern -> no_pos_arguments
+
+pos_arg_pattern: as_pattern ("," as_pattern)*
+keyws_arg_pattern: keyw_arg_pattern ("," keyw_arg_pattern)*
+keyw_arg_pattern: NAME "=" as_pattern
+
+
+
 suite: simple_stmt | _NEWLINE _INDENT stmt+ _DEDENT
 
 ?test: or_test ("if" or_test "else" test)?
      | lambdef
      | assign_expr
 
-assign_expr: NAME ":=" test
+assign_expr: name ":=" test
 
 ?test_nocond: or_test | lambdef_nocond
 
@@ -141,7 +191,7 @@ AWAIT: "await"
 
 ?atom_expr: atom_expr "(" [arguments] ")"      -> funccall
           | atom_expr "[" subscriptlist "]"  -> getitem
-          | atom_expr "." NAME               -> getattr
+          | atom_expr "." name               -> getattr
           | atom
 
 ?atom: "(" yield_expr ")"
@@ -153,7 +203,7 @@ AWAIT: "await"
      | "{" comprehension{key_value} "}" -> dict_comprehension
      | "{" _set_exprlist "}" -> set
      | "{" comprehension{test} "}" -> set_comprehension
-     | NAME -> var
+     | name -> var
      | number 
      | string_concat
      | "(" test ")"
@@ -186,7 +236,7 @@ key_value: test ":"  test
 
 _set_exprlist: test_or_star_expr (","  test_or_star_expr)* [","]
 
-classdef: "class" NAME ["(" [arguments] ")"] ":" suite
+classdef: "class" name ["(" [arguments] ")"] ":" suite
 
 
 
@@ -209,7 +259,7 @@ ASYNC: "async"
 ?comp_if: "if" test_nocond
 
 // not used in grammar, but may appear in "node" passed from Parser to Compiler
-encoding_decl: NAME
+encoding_decl: name
 
 yield_expr: "yield" [testlist]
           | "yield" "from" test -> yield_from
@@ -229,6 +279,7 @@ _NEWLINE: ( /\r?\n[\t ]*/ | COMMENT )+
 
 // Python terminals
 
+!name: NAME | "match" | "case"
 NAME: /[^\W\d]\w*/
 COMMENT: /#[^\n]*/
 


Test patch:
diff --git a/tests/test_python_grammar.py b/tests/test_python_grammar.py
index 9030798..837369e 100644
--- a/tests/test_python_grammar.py
+++ b/tests/test_python_grammar.py
@@ -1,9 +1,10 @@
+import inspect
+import textwrap
 from unittest import TestCase, main
 
 from lark import Lark
 from lark.indenter import PythonIndenter
-from lark.exceptions import UnexpectedCharacters, UnexpectedToken
-
+from lark.exceptions import UnexpectedCharacters, UnexpectedToken, ParseError
 
 valid_DEC_NUMBER = [
     "0",
@@ -169,12 +170,70 @@ invalid_number = [
 ]
 
 
+valid_match_statements = [
+    # constant and capture patterns
+    textwrap.dedent("""
+    match greeting:
+        case "":
+            print("Hello!")
+        case name:
+            print(f"Hi {name}!")
+    """),
+
+    # pattern unions
+    textwrap.dedent("""
+    match something:
+        case 0 | 1 | 2:
+            print("Small number")
+        case [] | [_]:
+            print("A short sequence")
+        case str() | bytes():
+            print("Something string-like")
+        case _:
+            print("Something else") 
+    """),
+
+    # guards
+    textwrap.dedent("""
+    match val:
+        case [x, y] if x > 0 and y > 0:
+            return f"A pair of {x} and {y}"
+        case [x, *other]:
+            return f"A sequence starting with {x}"
+        case int():
+            return f"Some integer"
+    """),
+
+    # "as" patterns
+    textwrap.dedent("""
+    match command.split():
+        case ["go", ("north" | "south" | "east" | "west") as direction]:
+            current_room = current_room.neighbor(direction)
+    """)
+]
+
+invalid_match_statements = [
+    # no cases
+    textwrap.dedent("""
+    match val:
+        pass
+    """),
+
+    # cases not indented relative to match
+    textwrap.dedent("""
+    match val:
+    case x:
+        pass
+    """)
+]
+
+
 class TestPythonParser(TestCase):
     @classmethod
     def setUpClass(cls):
         cls.python_parser = Lark.open_from_package(
             "lark", "python.lark", ("grammars",), parser='lalr',
-            postlex=PythonIndenter(), start=["number"])
+            postlex=PythonIndenter(), start=["number", "file_input"])
 
     def _test_parsed_is_this_terminal(self, text, terminal, start):
         tree = self.python_parser.parse(text, start=start)
@@ -183,6 +242,14 @@ class TestPythonParser(TestCase):
         self.assertEqual(token.type, terminal)
         self.assertEqual(token.value, text)
 
+    def _test_parsed_is_file_containing_only_this_statement(self, text, statement):
+        tree = self.python_parser.parse(text, start="file_input")
+        self.assertEqual(len(tree.children), 1)
+
+        statement_token = tree.children[0].data
+        self.assertEqual(statement_token.type, "RULE")
+        self.assertEqual(statement_token.value, statement)
+
     def test_DEC_NUMBER(self):
         for case in valid_DEC_NUMBER:
             self._test_parsed_is_this_terminal(case, "DEC_NUMBER", "number")
@@ -217,6 +284,30 @@ class TestPythonParser(TestCase):
             with self.assertRaises((UnexpectedCharacters, UnexpectedToken)):
                 self.python_parser.parse(case, start="number")
 
+    def test_valid_match_statement(self):
+        for case in valid_match_statements:
+            self._test_parsed_is_file_containing_only_this_statement(case, "match_stmt")
+
+    def test_invalid_match_statement(self):
+        for case in invalid_match_statements:
+            with self.assertRaises(ParseError):
+                self.python_parser.parse(case, start="file_input")
+
+    def test_assign_to_variable_named_match(self):
+        text = textwrap.dedent("""
+        match = re.match(pattern, string)
+        """)
+
+        self._test_parsed_is_file_containing_only_this_statement(text, "assign_stmt")
+
+    def test_assign_expr_with_variable_named_match(self):
+        text = textwrap.dedent("""
+        if match := re.match(pattern, string):
+            do_thing(match)
+        """)
+
+        self._test_parsed_is_file_containing_only_this_statement(text, "if_stmt")
+
 
 if __name__ == '__main__':
     main()