Repository: lark-parser/lark
Pull request #993 (2021-09-12 19:45:44)

Various backwards incompatible fixes for v1.0
Mostly obvious changes


Patch:
diff --git a/lark/CHANGELOG.md b/lark/CHANGELOG.md
new file mode 100644
index 0000000..9fdb072
--- /dev/null
+++ b/lark/CHANGELOG.md
@@ -0,0 +1,6 @@
+v1.0
+
+- `maybe_placeholders` is now True by default
+
+- `use_accepts` in `UnexpectedInput.match_examples()` is now True by default
+
diff --git a/lark/common.py b/lark/common.py
index 6c3962e..f5db5f7 100644
--- a/lark/common.py
+++ b/lark/common.py
@@ -1,14 +1,14 @@
-from copy import deepcopy
 from types import ModuleType
-from typing import Callable, Collection, Dict, Optional, TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from .lark import PostLex
+from copy import deepcopy
 
 from .utils import Serialize
 from .lexer import TerminalDef, Token
 
 ###{standalone
+from typing import Any, Callable, Collection, Dict, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .lark import PostLex
 
 _Callback = Callable[[Token], Token]
 
diff --git a/lark/exceptions.py b/lark/exceptions.py
index 2660e35..deea929 100644
--- a/lark/exceptions.py
+++ b/lark/exceptions.py
@@ -1,13 +1,15 @@
 from .utils import logger, NO_VALUE
-from typing import Dict, Iterable, Callable, Union, TypeVar, Tuple, Any, List, Set, Optional, Collection, TYPE_CHECKING
+
+
+###{standalone
+
+from typing import Dict, Iterable, Callable, Union, TypeVar, Tuple, Any, List, Set, Optional, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from .lexer import Token
     from .parsers.lalr_interactive_parser import InteractiveParser
     from .tree import Tree
 
-###{standalone
-
 class LarkError(Exception):
     pass
 
@@ -16,7 +18,7 @@ class ConfigurationError(LarkError, ValueError):
     pass
 
 
-def assert_config(value, options: Collection, msg='Got %r, expected one of %s'):
+def assert_config(value, options, msg='Got %r, expected one of %s'):
     if value not in options:
         raise ConfigurationError(msg % (value, options))
 
@@ -72,7 +74,11 @@ class UnexpectedInput(LarkError):
             after = text[pos:end].split(b'\n', 1)[0]
             return (before + after + b'\n' + b' ' * len(before.expandtabs()) + b'^\n').decode("ascii", "backslashreplace")
 
-    def match_examples(self, parse_fn: 'Callable[[str], Tree]', examples: Union[Dict[T, Iterable[str]], Iterable[Tuple[T, Iterable[str]]]], token_type_match_fallback: bool=False, use_accepts: bool=False) -> Optional[T]:
+    def match_examples(self, parse_fn: 'Callable[[str], Tree]', 
+                             examples: Union[Dict[T, Iterable[str]], Iterable[Tuple[T, Iterable[str]]]],
+                             token_type_match_fallback: bool=False,
+                             use_accepts: bool=True
+                         ) -> Optional[T]:
         """Allows you to detect what's wrong in the input text by matching
         against example errors.
 
@@ -87,8 +93,7 @@ class UnexpectedInput(LarkError):
         Parameters:
             parse_fn: parse function (usually ``lark_instance.parse``)
             examples: dictionary of ``{'example_string': value}``.
-            use_accepts: Recommended to call this with ``use_accepts=True``.
-                The default is ``False`` for backwards compatibility.
+            use_accepts: Recommended to keep this as ``use_accepts=True``.
         """
         assert self.state is not None, "Not supported for this exception"
 
@@ -104,7 +109,7 @@ class UnexpectedInput(LarkError):
                     parse_fn(malformed)
                 except UnexpectedInput as ut:
                     if ut.state == self.state:
-                        if use_accepts and hasattr(self, 'accepts') and ut.accepts != self.accepts:
+                        if use_accepts and hasattr(self, 'accepts') and hasattr(ut, 'accepts') and ut.accepts != self.accepts:
                             logger.debug("Different accepts with same state[%d]: %s != %s at example [%s][%s]" %
                                          (self.state, self.accepts, ut.accepts, i, j))
                             continue
diff --git a/lark/grammar.py b/lark/grammar.py
index 3d6f0ff..25aec17 100644
--- a/lark/grammar.py
+++ b/lark/grammar.py
@@ -1,9 +1,10 @@
-from typing import Optional, Tuple, ClassVar
 
 from .utils import Serialize
 
 ###{standalone
 
+from typing import Optional, Tuple, ClassVar
+
 class Symbol(Serialize):
     __slots__ = ('name',)
 
diff --git a/lark/grammars/python.lark b/lark/grammars/python.lark
index e73362d..684193d 100644
--- a/lark/grammars/python.lark
+++ b/lark/grammars/python.lark
@@ -10,8 +10,8 @@ DEC_NUMBER: /0|[1-9][\d_]*/i
 HEX_NUMBER.2: /0x[\da-f]*/i
 OCT_NUMBER.2: /0o[0-7]*/i
 BIN_NUMBER.2 : /0b[0-1]*/i
-FLOAT_NUMBER.2: /((\d+\.[\d_]*|\.[\d_]+)([Ee][-+]?\d+)?|\d+([Ee][-+]?\d+))/
-IMAG_NUMBER.2: /\d+[Jj]/ | FLOAT_NUMBER /[Jj]/
+FLOAT_NUMBER.2: /((\d+\.[\d_]*|\.[\d_]+)(e[-+]?\d+)?|\d+(e[-+]?\d+))/i
+IMAG_NUMBER.2: /\d+j/i | FLOAT_NUMBER "j"i
 
 
 // Comma-separated list (with an optional trailing comma)
diff --git a/lark/indenter.py b/lark/indenter.py
index 1a9e587..69a7ba4 100644
--- a/lark/indenter.py
+++ b/lark/indenter.py
@@ -1,18 +1,19 @@
 "Provides Indentation services for languages with indentation similar to Python"
 
 from abc import ABC, abstractmethod
-from typing import List, Iterator
 
 from .exceptions import LarkError
 from .lark import PostLex
 from .lexer import Token
 
 ###{standalone
+from typing import Tuple, List, Iterator, Optional
 
 class DedentError(LarkError):
     pass
 
 class Indenter(PostLex, ABC):
+
     paren_level: int
     indent_level: List[int]
 
@@ -74,31 +75,31 @@ class Indenter(PostLex, ABC):
     @property
     @abstractmethod
     def NL_type(self) -> str:
-        raise NotImplementedError()
+        ...
 
     @property
     @abstractmethod
     def OPEN_PAREN_types(self) -> List[str]:
-        raise NotImplementedError()
+        ...
 
     @property
     @abstractmethod
     def CLOSE_PAREN_types(self) -> List[str]:
-        raise NotImplementedError()
+        ...
 
     @property
     @abstractmethod
     def INDENT_type(self) -> str:
-        raise NotImplementedError()
+        ...
 
     @property
     @abstractmethod
     def DEDENT_type(self) -> str:
-        raise NotImplementedError()
+        ...
 
     @property
     @abstractmethod
     def tab_len(self) -> int:
-        raise NotImplementedError()
+        ...
 
 ###}
diff --git a/lark/lark.py b/lark/lark.py
index 206541a..380c32a 100644
--- a/lark/lark.py
+++ b/lark/lark.py
@@ -1,18 +1,7 @@
 from abc import ABC, abstractmethod
 import sys, os, pickle, hashlib
 import tempfile
-from typing import (
-    TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional,
-    Tuple, Iterable, IO, Any, TYPE_CHECKING
-)
-if TYPE_CHECKING:
-    from .parsers.lalr_interactive_parser import InteractiveParser
-    from .visitors import Transformer
-    if sys.version_info >= (3, 8):
-        from typing import Literal
-    else:
-        from typing_extensions import Literal
-        
+
 from .exceptions import ConfigurationError, assert_config, UnexpectedInput
 from .utils import Serialize, SerializeMemoizer, FS, isascii, logger
 from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource
@@ -32,7 +21,18 @@ except ImportError:
 
 
 ###{standalone
+from typing import (
+    TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional,
+    Tuple, Iterable, IO, Any, TYPE_CHECKING
+)
 
+if TYPE_CHECKING:
+    from .parsers.lalr_interactive_parser import InteractiveParser
+    from .visitors import Transformer
+    if sys.version_info >= (3, 8):
+        from typing import Literal
+    else:
+        from typing_extensions import Literal
 
 class PostLex(ABC):
     @abstractmethod
@@ -82,9 +82,8 @@ class LarkOptions(Serialize):
             Accepts ``False``, ``True``, or a callable, which will filter which nodes to ignore when propagating.
     maybe_placeholders
             When ``True``, the ``[]`` operator returns ``None`` when not matched.
-
             When ``False``,  ``[]`` behaves like the ``?`` operator, and returns no value at all.
-            (default= ``False``. Recommended to set to ``True``)
+            (default= ``True``)
     cache
             Cache the results of the Lark grammar analysis, for x2 to x3 faster loading. LALR only for now.
 
@@ -164,7 +163,7 @@ class LarkOptions(Serialize):
         'regex': False,
         'propagate_positions': False,
         'lexer_callbacks': {},
-        'maybe_placeholders': False,
+        'maybe_placeholders': True,
         'edit_terminals': None,
         'g_regex_flags': 0,
         'use_bytes': False,
@@ -303,8 +302,8 @@ class Lark(Serialize):
                 else:
                     if self.options.cache is not True:
                         raise ConfigurationError("cache argument must be bool or str")
-                    # Python2.7 doesn't support * syntax in tuples
-                    cache_fn = tempfile.gettempdir() + '/.lark_cache_%s_%s_%s.tmp' % ((cache_md5,) + sys.version_info[:2])
+                        
+                    cache_fn = tempfile.gettempdir() + '/.lark_cache_%s_%s_%s.tmp' % (cache_md5, *sys.version_info[:2])
 
                 if FS.exists(cache_fn):
                     logger.debug('Loading grammar from cache: %s', cache_fn)
@@ -368,7 +367,6 @@ class Lark(Serialize):
 
         if self.options.priority not in _VALID_PRIORITY_OPTIONS:
             raise ConfigurationError("invalid priority option: %r. Must be one of %r" % (self.options.priority, _VALID_PRIORITY_OPTIONS))
-        assert self.options.ambiguity not in ('resolve__antiscore_sum', ), 'resolve__antiscore_sum has been replaced with the option priority="invert"'
         if self.options.ambiguity not in _VALID_AMBIGUITY_OPTIONS:
             raise ConfigurationError("invalid ambiguity option: %r. Must be one of %r" % (self.options.ambiguity, _VALID_AMBIGUITY_OPTIONS))
 
@@ -387,7 +385,6 @@ class Lark(Serialize):
         self._terminals_dict = {t.name: t for t in self.terminals}
 
         # If the user asked to invert the priorities, negate them all here.
-        # This replaces the old 'resolve__antiscore_sum' option.
         if self.options.priority == 'invert':
             for rule in self.rules:
                 if rule.options.priority is not None:
diff --git a/lark/lexer.py b/lark/lexer.py
index 0b7bef8..aa34a30 100644
--- a/lark/lexer.py
+++ b/lark/lexer.py
@@ -3,13 +3,6 @@
 from abc import abstractmethod, ABC
 import re
 from contextlib import suppress
-from typing import (
-    TypeVar, Type, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any,
-    Pattern as REPattern, ClassVar, TYPE_CHECKING
-)
-from types import ModuleType
-if TYPE_CHECKING:
-    from .common import LexerConf
 
 from .utils import classify, get_regexp_width, Serialize
 from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken
@@ -17,6 +10,14 @@ from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken
 ###{standalone
 from copy import copy
 
+from types import ModuleType
+from typing import (
+    TypeVar, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any,
+    Pattern as REPattern, ClassVar, TYPE_CHECKING
+)
+
+if TYPE_CHECKING:
+    from .common import LexerConf
 
 class Pattern(Serialize, ABC):
 
@@ -217,7 +218,7 @@ class LineCounter:
 
         return self.char_pos == other.char_pos and self.newline_char == other.newline_char
 
-    def feed(self, token: Token, test_newline=True):
+    def feed(self, token, test_newline=True):
         """Consume a token and calculate the new line & column.
 
         As an optional optimization, set test_newline=False if token doesn't contain a newline.
@@ -319,7 +320,7 @@ class Scanner:
                 return m.group(0), type_from_index[m.lastindex]
 
 
-def _regexp_has_newline(r: str):
+def _regexp_has_newline(r):
     r"""Expressions that may indicate newlines in a regexp:
         - newlines (\n)
         - escaped newline (\\n)
@@ -358,7 +359,7 @@ class Lexer(ABC):
     """
     @abstractmethod
     def lex(self, lexer_state: LexerState, parser_state: Any) -> Iterator[Token]:
-        return NotImplemented
+        ...
 
     def make_lexer_state(self, text):
         line_ctr = LineCounter(b'\n' if isinstance(text, bytes) else '\n')
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index c0503e6..4d73e61 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -6,7 +6,6 @@ from collections import namedtuple
 from copy import copy, deepcopy
 import pkgutil
 from ast import literal_eval
-from numbers import Integral
 from contextlib import suppress
 from typing import List, Tuple, Union, Callable, Dict, Optional
 
@@ -1067,8 +1066,7 @@ class GrammarBuilder:
         if self._is_term(name):
             if options is None:
                 options = 1
-            # if we don't use Integral here, we run into python2.7/python3 problems with long vs int
-            elif not isinstance(options, Integral):
+            elif not isinstance(options, int):
                 raise GrammarError("Terminal require a single int as 'options' (e.g. priority), got %s" % (type(options),))
         else:
             if options is None:
diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py
index 4342b24..e95003a 100644
--- a/lark/parse_tree_builder.py
+++ b/lark/parse_tree_builder.py
@@ -1,5 +1,3 @@
-from typing import List
-
 from .exceptions import GrammarError, ConfigurationError
 from .lexer import Token
 from .tree import Tree
@@ -153,7 +151,7 @@ def _should_expand(sym):
     return not sym.is_term and sym.name.startswith('_')
 
 
-def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices: List[bool]):
+def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices):
     # Prepare empty_indices as: How many Nones to insert at each index?
     if _empty_indices:
         assert _empty_indices.count(False) == len(expansion)
diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py
index 37d59ae..475f70d 100644
--- a/lark/parser_frontends.py
+++ b/lark/parser_frontends.py
@@ -32,13 +32,20 @@ class MakeParsingFrontend:
         self.parser_type = parser_type
         self.lexer_type = lexer_type
 
+    def __call__(self, lexer_conf, parser_conf, options):
+        assert isinstance(lexer_conf, LexerConf)
+        assert isinstance(parser_conf, ParserConf)
+        parser_conf.parser_type = self.parser_type
+        lexer_conf.lexer_type = self.lexer_type
+        return ParsingFrontend(lexer_conf, parser_conf, options)
+
     def deserialize(self, data, memo, lexer_conf, callbacks, options):
         parser_conf = ParserConf.deserialize(data['parser_conf'], memo)
         parser = LALR_Parser.deserialize(data['parser'], memo, callbacks, options.debug)
         parser_conf.callbacks = callbacks
         return ParsingFrontend(lexer_conf, parser_conf, options, parser=parser)
 
-    # ... Continued later in the module
+
 
 
 class ParsingFrontend(Serialize):
@@ -230,12 +237,3 @@ class CYK_FrontEnd:
 
     def _apply_callback(self, tree):
         return self.callbacks[tree.rule](tree.children)
-
-
-class MakeParsingFrontend(MakeParsingFrontend):
-    def __call__(self, lexer_conf, parser_conf, options):
-        assert isinstance(lexer_conf, LexerConf)
-        assert isinstance(parser_conf, ParserConf)
-        parser_conf.parser_type = self.parser_type
-        lexer_conf.lexer_type = self.lexer_type
-        return ParsingFrontend(lexer_conf, parser_conf, options)
diff --git a/lark/tools/standalone.py b/lark/tools/standalone.py
index 7282699..1cc8f81 100644
--- a/lark/tools/standalone.py
+++ b/lark/tools/standalone.py
@@ -25,13 +25,6 @@
 #
 
 from abc import ABC, abstractmethod
-from collections.abc import Sequence
-from types import ModuleType
-from typing import (
-    TypeVar, Generic, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any,
-    Union, Iterable, IO, TYPE_CHECKING,
-    Pattern as REPattern, ClassVar, Set,
-)
 ###}
 
 import sys
diff --git a/lark/tree.py b/lark/tree.py
index 1ca0c62..90ec0fe 100644
--- a/lark/tree.py
+++ b/lark/tree.py
@@ -1,4 +1,3 @@
-
 try:
     from future_builtins import filter  # type: ignore
 except ImportError:
@@ -7,7 +6,10 @@ except ImportError:
 import sys
 from copy import deepcopy
 
-from typing import List, Callable, Iterator, Union, Optional, TYPE_CHECKING
+
+###{standalone
+from collections import OrderedDict
+from typing import List, Callable, Iterator, Union, Optional, Any, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from .lexer import TerminalDef
@@ -16,9 +18,6 @@ if TYPE_CHECKING:
     else:
         from typing_extensions import Literal
 
-###{standalone
-from collections import OrderedDict
-
 class Meta:
 
     empty: bool
diff --git a/lark/utils.py b/lark/utils.py
index ffaa593..2b6e3b6 100644
--- a/lark/utils.py
+++ b/lark/utils.py
@@ -241,7 +241,7 @@ def combine_alternatives(lists):
 try:
     import atomicwrites
 except ImportError:
-    atomicwrites = None  # type: ignore
+    atomicwrites = None
 
 class FS:
     exists = os.path.exists
diff --git a/lark/visitors.py b/lark/visitors.py
index 954886a..60923b3 100644
--- a/lark/visitors.py
+++ b/lark/visitors.py
@@ -1,4 +1,3 @@
-from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union, Optional
 from abc import ABC
 from functools import wraps
 
@@ -9,6 +8,7 @@ from .lexer import Token
 
 ###{standalone
 from inspect import getmembers, getmro
+from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union, Optional
 
 _T = TypeVar('_T')
 _R = TypeVar('_R')
@@ -385,7 +385,7 @@ def _vargs_inline(f, _data, children, _meta):
 def _vargs_meta_inline(f, _data, children, meta):
     return f(meta, *children)
 def _vargs_meta(f, _data, children, meta):
-    return f(children, meta)   # TODO swap these for consistency? Backwards incompatible!
+    return f(meta, children)
 def _vargs_tree(f, data, children, meta):
     return f(Tree(data, children, meta))
 


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
index ebc6152..dab69f7 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -208,11 +208,11 @@ class TestParsers(unittest.TestCase):
 
         @v_args(meta=True)
         class T1(Transformer):
-            def a(self, children, meta):
+            def a(self, meta, children):
                 assert not children
                 return meta.line
 
-            def start(self, children, meta):
+            def start(self, meta, children):
                 return children
 
         @v_args(meta=True, inline=True)
diff --git a/tests/test_reconstructor.py b/tests/test_reconstructor.py
index e2f2dbe..4df1cb9 100644
--- a/tests/test_reconstructor.py
+++ b/tests/test_reconstructor.py
@@ -183,8 +183,8 @@ class TestReconstructor(TestCase):
         keyword x += y
         """
 
-        l1 = Lark(g1, parser='lalr')
-        l2 = Lark(g2, parser='lalr')
+        l1 = Lark(g1, parser='lalr', maybe_placeholders=False)
+        l2 = Lark(g2, parser='lalr', maybe_placeholders=False)
         r = Reconstructor(l2)
 
         tree = l1.parse(code)
diff --git a/tests/test_tools.py b/tests/test_tools.py
index fd42b1c..7a732d1 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -24,7 +24,7 @@ class TestStandalone(TestCase):
         standalone.gen_standalone(Lark(grammar, parser='lalr'), out=code_buf, compress=compress)
         code = code_buf.getvalue()
 
-        context = {'__doc__': None, '__name__': 'test_standalone'}
+        context = {'__doc__': None}
         exec(code, context)
         return context