Repository: lark-parser/lark
Pull request #1265 (2023-03-09 14:12:09)

Docs: Added section about strict-mode to how-to-use


Patch:
diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml
index 0b5d5d5..d078850 100644
--- a/.github/workflows/codecov.yml
+++ b/.github/workflows/codecov.yml
@@ -11,7 +11,7 @@ jobs:
       PYTHON: '3.7'
     steps:
     - uses: actions/checkout@v3
-      name: Download with submodules
+    - name: Download submodules
       with:
         submodules: recursive
     - name: Setup Python
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4dfc25f..2852d8e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -15,7 +15,7 @@
 repos:
 # Standard hooks
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: "v4.4.0"
+  rev: "v4.2.0"
   hooks:
     - id: check-added-large-files
     - id: check-case-conflict
@@ -30,9 +30,3 @@ repos:
     - id: requirements-txt-fixer
     - id: trailing-whitespace
       exclude: '(^tests/.*\.lark|\.svg)$'
-
-- repo: https://github.com/codespell-project/codespell
-  rev: v2.2.2
-  hooks:
-    - id: codespell
-      args: ["-L", "nd,iif,ot,datas"]
diff --git a/docs/how_to_use.md b/docs/how_to_use.md
index e10be30..85558ac 100644
--- a/docs/how_to_use.md
+++ b/docs/how_to_use.md
@@ -74,9 +74,13 @@ Note 1: Interegular currently only runs when the lexer is `basic` or `contextual
 
 Note 2: Some advanced regex features, such as lookahead and lookbehind, may prevent interegular from detecting existing collisions.
 
-### LALR
+### Shift/Reduce collisions
 
-By default Lark silently resolves Shift/Reduce conflicts as Shift. To enable warnings pass `debug=True`. To get the messages printed you have to configure the `logger` beforehand. For example:
+By default Lark automatically resolves Shift/Reduce conflicts as Shift. It produces notifications as debug messages.
+
+when users pass `debug=True`, those notifications are written as warnings.
+
+Either way, to get the messages printed you have to configure the `logger` beforehand. For example:
 
 ```python
 import logging
@@ -90,6 +94,43 @@ as: a*
 a: "a"
 '''
 p = Lark(collision_grammar, parser='lalr', debug=True)
+# Shift/Reduce conflict for terminal A: (resolving as shift)
+#  * <as : >
+# Shift/Reduce conflict for terminal A: (resolving as shift)
+#  * <as : __as_star_0>
+```
+
+### Strict-Mode
+
+Lark, by default, accepts grammars with unresolved Shift/Reduce collisions (which it always resolves to shift), and regex collisions.
+
+Strict-mode allows users to validate that their grammars don't contain these collisions.
+
+When Lark is initialized with `strict=True`, it raises an exception on any Shift/Reduce or regex collision.
+
+If `interegular` isn't installed, an exception is thrown.
+
+When using strict-mode, users will be expected to resolve their collisions manually:
+
+- To resolve Shift/Reduce collisions, adjust the priority weights of the rules involved, until there are no more collisions.
+
+- To resolve regex collisions, change the involved regexes so that they can no longer both match the same input (Lark provides an example).
+
+Strict-mode only applies to LALR for now.
+
+```python
+from lark import Lark
+
+collision_grammar = '''
+start: as as
+as: a*
+a: "a"
+'''
+p = Lark(collision_grammar, parser='lalr', strict=True)
+
+# Traceback (most recent call last):
+#   ...
+# lark.exceptions.GrammarError: Shift/Reduce conflict for terminal A. [strict-mode]
 ```
 
 ## Tools
diff --git a/lark/common.py b/lark/common.py
index 870b515..d716add 100644
--- a/lark/common.py
+++ b/lark/common.py
@@ -38,10 +38,8 @@ class LexerConf(Serialize):
     skip_validation: bool
     use_bytes: bool
     lexer_type: Optional[_LexerArgType]
-    strict: bool
 
-    def __init__(self, terminals: Collection[TerminalDef], re_module: ModuleType, ignore: Collection[str]=(), postlex: 'Optional[PostLex]'=None,
-                 callbacks: Optional[Dict[str, _Callback]]=None, g_regex_flags: int=0, skip_validation: bool=False, use_bytes: bool=False, strict: bool=False):
+    def __init__(self, terminals: Collection[TerminalDef], re_module: ModuleType, ignore: Collection[str]=(), postlex: 'Optional[PostLex]'=None, callbacks: Optional[Dict[str, _Callback]]=None, g_regex_flags: int=0, skip_validation: bool=False, use_bytes: bool=False):
         self.terminals = terminals
         self.terminals_by_name = {t.name: t for t in self.terminals}
         assert len(self.terminals) == len(self.terminals_by_name)
@@ -52,7 +50,6 @@ class LexerConf(Serialize):
         self.re_module = re_module
         self.skip_validation = skip_validation
         self.use_bytes = use_bytes
-        self.strict = strict
         self.lexer_type = None
 
     def _deserialize(self):
diff --git a/lark/lark.py b/lark/lark.py
index e2bec74..c93e9e1 100644
--- a/lark/lark.py
+++ b/lark/lark.py
@@ -20,7 +20,7 @@ if TYPE_CHECKING:
 
 from .exceptions import ConfigurationError, assert_config, UnexpectedInput
 from .utils import Serialize, SerializeMemoizer, FS, isascii, logger
-from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource, sha256_digest
+from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource, md5_digest
 from .tree import Tree
 from .common import LexerConf, ParserConf, _ParserArgType, _LexerArgType
 
@@ -54,7 +54,6 @@ class LarkOptions(Serialize):
 
     start: List[str]
     debug: bool
-    strict: bool
     transformer: 'Optional[Transformer]'
     propagate_positions: Union[bool, str]
     maybe_placeholders: bool
@@ -82,8 +81,6 @@ class LarkOptions(Serialize):
     debug
             Display debug information and extra warnings. Use only when debugging (Default: ``False``)
             When used with Earley, it generates a forest graph as "sppf.png", if 'dot' is installed.
-    strict
-            Throw an exception on any potential ambiguity, including shift/reduce conflicts, and regex collisions.
     transformer
             Applies the transformer to every parse tree (equivalent to applying it after the parse, but faster)
     propagate_positions
@@ -159,7 +156,6 @@ class LarkOptions(Serialize):
     # - Potentially in `lark.tools.__init__`, if it makes sense, and it can easily be passed as a cmd argument
     _defaults: Dict[str, Any] = {
         'debug': False,
-        'strict': False,
         'keep_all_tokens': False,
         'tree_class': None,
         'cache': False,
@@ -292,7 +288,7 @@ class Lark(Serialize):
             grammar = read()
 
         cache_fn = None
-        cache_sha256 = None
+        cache_md5 = None
         if isinstance(grammar, str):
             self.source_grammar = grammar
             if self.options.use_bytes:
@@ -307,7 +303,7 @@ class Lark(Serialize):
                 options_str = ''.join(k+str(v) for k, v in options.items() if k not in unhashable)
                 from . import __version__
                 s = grammar + options_str + __version__ + str(sys.version_info[:2])
-                cache_sha256 = sha256_digest(s)
+                cache_md5 = md5_digest(s)
 
                 if isinstance(self.options.cache, str):
                     cache_fn = self.options.cache
@@ -323,7 +319,7 @@ class Lark(Serialize):
                         # specific reason - we just want a username.
                         username = "unknown"
 
-                    cache_fn = tempfile.gettempdir() + "/.lark_cache_%s_%s_%s_%s.tmp" % (username, cache_sha256, *sys.version_info[:2])
+                    cache_fn = tempfile.gettempdir() + "/.lark_cache_%s_%s_%s_%s.tmp" % (username, cache_md5, *sys.version_info[:2])
 
                 old_options = self.options
                 try:
@@ -332,9 +328,9 @@ class Lark(Serialize):
                         # Remove options that aren't relevant for loading from cache
                         for name in (set(options) - _LOAD_ALLOWED_OPTIONS):
                             del options[name]
-                        file_sha256 = f.readline().rstrip(b'\n')
+                        file_md5 = f.readline().rstrip(b'\n')
                         cached_used_files = pickle.load(f)
-                        if file_sha256 == cache_sha256.encode('utf8') and verify_used_files(cached_used_files):
+                        if file_md5 == cache_md5.encode('utf8') and verify_used_files(cached_used_files):
                             cached_parser_data = pickle.load(f)
                             self._load(cached_parser_data, **options)
                             return
@@ -428,7 +424,7 @@ class Lark(Serialize):
         # TODO Deprecate lexer_callbacks?
         self.lexer_conf = LexerConf(
                 self.terminals, re_module, self.ignore_tokens, self.options.postlex,
-                self.options.lexer_callbacks, self.options.g_regex_flags, use_bytes=self.options.use_bytes, strict=self.options.strict
+                self.options.lexer_callbacks, self.options.g_regex_flags, use_bytes=self.options.use_bytes
             )
 
         if self.options.parser:
@@ -440,8 +436,8 @@ class Lark(Serialize):
             logger.debug('Saving grammar to cache: %s', cache_fn)
             try:
                 with FS.open(cache_fn, 'wb') as f:
-                    assert cache_sha256 is not None
-                    f.write(cache_sha256.encode('utf8') + b'\n')
+                    assert cache_md5 is not None
+                    f.write(cache_md5.encode('utf8') + b'\n')
                     pickle.dump(used_files, f)
                     self.save(f, _LOAD_ALLOWED_OPTIONS)
             except IOError as e:
diff --git a/lark/lexer.py b/lark/lexer.py
index 14b7b6b..f14ff06 100644
--- a/lark/lexer.py
+++ b/lark/lexer.py
@@ -499,14 +499,11 @@ class BasicLexer(Lexer):
                         # Mark this pair to not repeat warnings when multiple different BasicLexers see the same collision
                         comparator.mark(a, b)
 
-                        # Notify the user
-                        message = f"Collision between Terminals {a.name} and {b.name}. "
+                        # leave it as a warning for the moment
+                        # raise LexError("Collision between Terminals %s and %s" % (a.name, b.name))
                         example = comparator.get_example_overlap(a, b).format_multiline()
-                        if conf.strict:
-                            raise LexError(f"{message}\n{example}")
-                        logger.warning("%s The lexer will choose between them arbitrarily.\n%s", message, example)
-            elif conf.strict:
-                raise LexError("interegular must be installed for strict mode. Use `pip install 'lark[interegular]'`.")
+                        logger.warning(f"Collision between Terminals {a.name} and {b.name}. "
+                                       f"The lexer will choose between them arbitrarily.\n" + example)
 
         # Init
         self.newline_types = frozenset(t.name for t in terminals if _regexp_has_newline(t.pattern.to_regexp()))
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index 90401ad..3018133 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -1314,7 +1314,7 @@ class GrammarBuilder:
             except IOError:
                 continue
             else:
-                h = sha256_digest(text)
+                h = md5_digest(text)
                 if self.used_files.get(joined_path, h) != h:
                     raise RuntimeError("Grammar file was changed during importing")
                 self.used_files[joined_path] = h
@@ -1393,7 +1393,7 @@ def verify_used_files(file_hashes):
         if text is None: # We don't know how to load the path. ignore it.
             continue
 
-        current = sha256_digest(text)
+        current = md5_digest(text)
         if old != current:
             logger.info("File %r changed, rebuilding Parser" % path)
             return False
@@ -1411,13 +1411,13 @@ def load_grammar(grammar, source, import_paths, global_keep_all_tokens):
     return builder.build(), builder.used_files
 
 
-def sha256_digest(s: str) -> str:
-    """Get the sha256 digest of a string
+def md5_digest(s: str) -> str:
+    """Get the md5 digest of a string
 
     Supports the `usedforsecurity` argument for Python 3.9+ to allow running on
     a FIPS-enabled system.
     """
     if sys.version_info >= (3, 9):
-        return hashlib.sha256(s.encode('utf8'), usedforsecurity=False).hexdigest()
+        return hashlib.md5(s.encode('utf8'), usedforsecurity=False).hexdigest()
     else:
-        return hashlib.sha256(s.encode('utf8')).hexdigest()
+        return hashlib.md5(s.encode('utf8')).hexdigest()
diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py
index d16b1ab..4e28e36 100644
--- a/lark/parser_frontends.py
+++ b/lark/parser_frontends.py
@@ -145,9 +145,8 @@ def create_contextual_lexer(lexer_conf, parser, postlex, options):
 
 def create_lalr_parser(lexer_conf, parser_conf, options=None):
     debug = options.debug if options else False
-    strict = options.strict if options else False
     cls = (options and options._plugins.get('LALR_Parser')) or LALR_Parser
-    return cls(parser_conf, debug=debug, strict=strict)
+    return cls(parser_conf, debug=debug)
 
 _parser_creators['lalr'] = create_lalr_parser
 
diff --git a/lark/parsers/grammar_analysis.py b/lark/parsers/grammar_analysis.py
index bdbadf3..b526e47 100644
--- a/lark/parsers/grammar_analysis.py
+++ b/lark/parsers/grammar_analysis.py
@@ -122,9 +122,8 @@ def calculate_sets(rules):
 
 
 class GrammarAnalyzer:
-    def __init__(self, parser_conf, debug=False, strict=False):
+    def __init__(self, parser_conf, debug=False):
         self.debug = debug
-        self.strict = strict
 
         root_rules = {start: Rule(NonTerminal('$root_' + start), [NonTerminal(start), Terminal('$END')])
                       for start in parser_conf.start}
diff --git a/lark/parsers/lalr_analysis.py b/lark/parsers/lalr_analysis.py
index b0e5529..06b0665 100644
--- a/lark/parsers/lalr_analysis.py
+++ b/lark/parsers/lalr_analysis.py
@@ -131,8 +131,8 @@ def traverse(x, S, N, X, R, G, F):
 
 
 class LALR_Analyzer(GrammarAnalyzer):
-    def __init__(self, parser_conf, debug=False, strict=False):
-        GrammarAnalyzer.__init__(self, parser_conf, debug, strict)
+    def __init__(self, parser_conf, debug=False):
+        GrammarAnalyzer.__init__(self, parser_conf, debug)
         self.nonterminal_transitions = []
         self.directly_reads = defaultdict(set)
         self.reads = defaultdict(set)
@@ -256,18 +256,10 @@ class LALR_Analyzer(GrammarAnalyzer):
                         rules = [best[1]]
                     else:
                         reduce_reduce.append((state, la, rules))
-                        continue
-
-                rule ,= rules
                 if la in actions:
-                    if self.strict:
-                        raise GrammarError(f"Shift/Reduce conflict for terminal {la.name}. [strict-mode]\n ")
-                    elif self.debug:
+                    if self.debug:
                         logger.warning('Shift/Reduce conflict for terminal %s: (resolving as shift)', la.name)
-                        logger.warning(' * %s', rule)
-                    else:
-                        logger.debug('Shift/Reduce conflict for terminal %s: (resolving as shift)', la.name)
-                        logger.debug(' * %s', rule)
+                        logger.warning(' * %s', list(rules)[0])
                 else:
                     actions[la] = (Reduce, list(rules)[0])
             m[state] = { k.name: v for k, v in actions.items() }
diff --git a/lark/parsers/lalr_parser.py b/lark/parsers/lalr_parser.py
index 7786e3f..5cf6ca4 100644
--- a/lark/parsers/lalr_parser.py
+++ b/lark/parsers/lalr_parser.py
@@ -14,8 +14,8 @@ from lark.exceptions import UnexpectedCharacters, UnexpectedInput, UnexpectedTok
 ###{standalone
 
 class LALR_Parser(Serialize):
-    def __init__(self, parser_conf, debug=False, strict=False):
-        analysis = LALR_Analyzer(parser_conf, debug=debug, strict=strict)
+    def __init__(self, parser_conf, debug=False):
+        analysis = LALR_Analyzer(parser_conf, debug=debug)
         analysis.compute_lalr()
         callbacks = parser_conf.callbacks
 
diff --git a/setup.py b/setup.py
index 7c56ddd..52121a2 100644
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,7 @@ setup(
         "regex": ["regex"],
         "nearley": ["js2py"],
         "atomic_cache": ["atomicwrites"],
-        "interegular": ["interegular>=0.2.6,<0.3.0"],
+        "interegular": ["interegular==0.2.4"],
     },
 
     package_data = {'': ['*.md', '*.lark'], 'lark': ['py.typed']},


Test patch:
diff --git a/test-requirements.txt b/test-requirements.txt
index 3625287..c41c984 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,3 +1,3 @@
-interegular>=0.2.6,<0.3.0
+interegular==0.2.4
 Js2Py==0.68
 regex
diff --git a/tests/test_logger.py b/tests/test_logger.py
index e52a2c8..0a0563e 100644
--- a/tests/test_logger.py
+++ b/tests/test_logger.py
@@ -41,7 +41,7 @@ class Testlogger(TestCase):
         self.assertIn("A", log)
 
     def test_non_debug(self):
-        logger.setLevel(logging.WARNING)
+        logger.setLevel(logging.DEBUG)
         collision_grammar = '''
         start: as as
         as: a*
diff --git a/tests/test_parser.py b/tests/test_parser.py
index a749689..3ebb0eb 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -2625,25 +2625,6 @@ def _make_parser_test(LEXER, PARSER):
             b = parser.parse(s)
             assert a == b
 
-        @unittest.skipIf(PARSER!='lalr', "strict mode is only supported in lalr for now")
-        def test_strict(self):
-            # Test regex collision
-            grammar = r"""
-            start: A | B
-
-            A: /e?rez/
-            B: /erez?/
-            """
-
-            self.assertRaises(LexError, _Lark, grammar, strict=True)
-
-            # Test shift-reduce collision
-            grammar = r"""
-            start: a "."
-            a: "."+
-            """
-            self.assertRaises(GrammarError, _Lark, grammar, strict=True)
-
 
     _NAME = "Test" + PARSER.capitalize() + LEXER.capitalize()
     _TestParser.__name__ = _NAME